From 087fe0fe2f67ebfffb668ed53fa65095cfba137d Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Thu, 10 Sep 2020 15:27:53 +0200 Subject: [PATCH 001/116] peak fitting Fit one peak from ccl as combination of gaussian, lorentian and backgroud --- pyzebra/fit2.py | 151 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 pyzebra/fit2.py diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py new file mode 100644 index 0000000..39929a8 --- /dev/null +++ b/pyzebra/fit2.py @@ -0,0 +1,151 @@ +from lmfit import minimize, Parameters, Model +from lmfit.models import LinearModel, LorentzianModel, GaussianModel +import matplotlib.pyplot as plt +from scipy.integrate import simps +import scipy as sc +from scipy import integrate +import numpy as np +from time import sleep + + +def fitccl(data, keys, guess, vary, constraints_min, constraints_max, numfit_min=None, numfit_max=None): + """Made for fitting of ccl date where 1 peak is expected. Allows for combination of gaussian, lorentzian and linear model combination + + + :param data: dictionary after peak fining + :param keys: name of the measurement in the data dict (i.e. M123) + :param guess: initial guess for the fitting, if none, some values are added automatically in order (see below) + :param vary: True if parameter can vary during fitting, False if it to be fixed + :param numfit_min: minimal value on x axis for numerical integration - if none is centre of gaussian minus 3 sigma + :param numfit_max: maximal value on x axis for numerical integration - if none is centre of gaussian plus 3 sigma + :param constraints_min: min constranits value for fit + :param constraints_max: max constranits value for fit + + :return data dict with additional values + + order for guess, vary, constraints_min, constraints_max + [Gaussian centre, Gaussian sigma, Gaussian amplitude, Lorentzian centre, Lorentzian sigma, Lorentzian amplitude, background slope, background intercept] + examples: + guess = [None, None, 100, None, None, None, 0, None] + vary = [True, True, True, True, False, True, True, True] + constraints_min = [23, None, 50, None, None, None, 0, 0] + constraints_min = [80, None, 1000, None, None, None, 0, 100] + """ + + if len(data["Measurements"][str(keys)]["peak_indexes"]) == 1: + x = list(data["Measurements"][str(keys)]["omega"]) + y = list(data["Measurements"][str(keys)]["counts"]) + peak_index = data["Measurements"][str(keys)]["peak_indexes"] + peak_height = data["Measurements"][str(keys)]["peak_heights"] + print('before', constraints_min) + guess[0] = x[int(peak_index)] if guess[0] is None else guess[0] + guess[1] = 0.1 if guess[1] is None else guess[1] + guess[2] = float(peak_height/10) if guess[2]is None else float(guess[2]) + guess[3] = x[int(peak_index)] if guess[3] is None else guess[3] + guess[4] = 2*guess[1] if guess[4] is None else guess[4] + guess[5] = float(peak_height/10) if guess[5] is None else float(guess[5]) + guess[6] = 0 if guess[6] is None else guess[6] + guess[7] = np.median(x) if guess[7] is None else guess[7] + constraints_min[0] = np.min(x) if constraints_min[0] is None else constraints_min[0] + constraints_min[3] = np.min(x) if constraints_min[3] is None else constraints_min[3] + constraints_max[0] = np.max(x) if constraints_max[0] is None else constraints_max[0] + constraints_max[3] = np.max(x) if constraints_max[3] is None else constraints_max[3] + print('key', keys) + + print('after', constraints_min) + + + def find_nearest(array, value): + array = np.asarray(array) + idx = (np.abs(array - value)).argmin() + return idx + + def gaussian(x, g_cen, g_width, g_amp): + """1-d gaussian: gaussian(x, amp, cen, wid)""" + return (g_amp / (np.sqrt(2.0 * np.pi) * g_width)) * np.exp(-(x - g_cen) ** 2 / (2 * g_width ** 2)) + + def lorentzian(x, l_cen, l_width, l_amp): + """1d lorentzian""" + return (l_amp / (1 + ((1 * x - l_cen) / l_width) ** 2)) / (np.pi * l_width) + + def background(x, slope, intercept): + """background""" + return slope*x + intercept + + mod = Model(gaussian) + Model(lorentzian) + Model(background) + params = Parameters() + params.add_many(('g_cen', x[int(peak_index)], bool(vary[0]), np.min(x), np.max(x), None, None), + ('g_width', guess[1], bool(vary[1]), constraints_min[1], constraints_max[1], None, None), + ('g_amp', guess[2], bool(vary[2]), constraints_min[2], constraints_max[2], None, None), + ('l_cen', guess[3], bool(vary[3]), np.min(x), np.max(x), None, None), + ('l_width', guess[4], bool(vary[4]), constraints_min[4], constraints_max[4], None, None), + ('l_amp', guess[5], bool(vary[5]), constraints_min[5], constraints_max[5], None, None), + ('slope', guess[6], bool(vary[6]), constraints_min[6], constraints_max[6], None, None), + ('intercept', guess[7], bool(vary[7]), constraints_min[7], constraints_max[7], None, None)) + + result = mod.fit(y, params, x=x) + print('Chi-sqr: ', result.chisqr) + + comps = result.eval_components() + + gauss_3sigmamin = find_nearest(x, result.params['g_cen'].value-3*result.params['g_width'].value) + gauss_3sigmamax = find_nearest(x, result.params['g_cen'].value+3*result.params['g_width'].value) + numfit_min = gauss_3sigmamin if numfit_min is None else find_nearest(x, numfit_min) + numfit_max = gauss_3sigmamax if numfit_max is None else find_nearest(x, numfit_max) + print(numfit_max, numfit_min) + if x[numfit_min] < np.min(x): + numfit_min = gauss_3sigmamin + print('Minimal integration value outside of x range') + elif x[numfit_min] >= x[numfit_max]: + numfit_min = gauss_3sigmamin + print('Minimal integration value higher than maximal') + else: + pass + if x[numfit_max] > np.max(x): + numfit_max = gauss_3sigmamax + print('Maximal integration value outside of x range') + elif x[numfit_max] <= x[numfit_min]: + numfit_max = gauss_3sigmamax + print('Maximal integration value lower than minimal') + else: + pass + print(result.params['g_width'].value) + print(result.params['g_cen'].value) + num_int_area = simps(y[numfit_min:numfit_max], x[numfit_min:numfit_max]) + num_int_bacground = integrate.quad(background, numfit_min, numfit_max, args=(result.params['slope'].value,result.params['intercept'].value)) + + plt.plot(x, y, 'b', label='Original data') + plt.plot(x, comps['gaussian'], 'r--', label='Gaussian component') + plt.fill_between(x, comps['gaussian'], facecolor="red", alpha=0.4) + plt.plot(x, comps['lorentzian'], 'b--', label='Lorentzian component') + plt.fill_between(x, comps['lorentzian'], facecolor="blue", alpha=0.4) + plt.plot(x, comps['background'], 'g--', label='Line component') + plt.fill_between(x, comps['background'], facecolor="green", alpha=0.4) + #plt.plot(x[numfit_min:numfit_max],y[numfit_min:numfit_max], 'vy', markersize=7) + plt.fill_between(x[numfit_min:numfit_max], y[numfit_min:numfit_max], facecolor="yellow", alpha=0.4, label='Integrated area') + #plt.plot(x, result.init_fit, 'k--', label='initial fit') + plt.plot(x, result.best_fit, 'k-', label='Best fit') + plt.title('%s \n Gaussian: centre = %9.4f, width = %9.4f, amp = %9.4f \n' + 'Lorentzian: centre, %9.4f, width = %9.4f, amp = %9.4f \n' + 'background: slope = %9.4f, intercept = %9.4f, int_area %9.4f' % (keys, result.params['g_cen'].value, result.params['g_width'].value, + result.params['g_amp'].value, result.params['l_cen'].value, result.params['l_width'].value, + result.params['l_amp'].value, result.params['slope'].value, result.params['intercept'].value, num_int_area)) + + plt.legend(loc='best') + plt.show() + d = {} + for pars in result.params: + d[str(pars)] = (result.params[str(pars)].value, result.params[str(pars)].vary) + + d["int_area"] = num_int_area + d["int_background"] = num_int_bacground + d["full_report"] = result.fit_report() + data["Measurements"][str(keys)]["fit"] = d + + return data + + else: + return print('NO PEAK or more than 1 peak') + + + From fe5ed4b9870bbfd0f93b7d6c075703545d01fb3d Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 11 Sep 2020 15:02:34 +0200 Subject: [PATCH 002/116] Update fit2.py --- pyzebra/fit2.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 39929a8..967bb41 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -1,12 +1,9 @@ from lmfit import minimize, Parameters, Model from lmfit.models import LinearModel, LorentzianModel, GaussianModel -import matplotlib.pyplot as plt from scipy.integrate import simps import scipy as sc from scipy import integrate import numpy as np -from time import sleep - def fitccl(data, keys, guess, vary, constraints_min, constraints_max, numfit_min=None, numfit_max=None): """Made for fitting of ccl date where 1 peak is expected. Allows for combination of gaussian, lorentzian and linear model combination @@ -113,26 +110,6 @@ def fitccl(data, keys, guess, vary, constraints_min, constraints_max, numfit_min print(result.params['g_cen'].value) num_int_area = simps(y[numfit_min:numfit_max], x[numfit_min:numfit_max]) num_int_bacground = integrate.quad(background, numfit_min, numfit_max, args=(result.params['slope'].value,result.params['intercept'].value)) - - plt.plot(x, y, 'b', label='Original data') - plt.plot(x, comps['gaussian'], 'r--', label='Gaussian component') - plt.fill_between(x, comps['gaussian'], facecolor="red", alpha=0.4) - plt.plot(x, comps['lorentzian'], 'b--', label='Lorentzian component') - plt.fill_between(x, comps['lorentzian'], facecolor="blue", alpha=0.4) - plt.plot(x, comps['background'], 'g--', label='Line component') - plt.fill_between(x, comps['background'], facecolor="green", alpha=0.4) - #plt.plot(x[numfit_min:numfit_max],y[numfit_min:numfit_max], 'vy', markersize=7) - plt.fill_between(x[numfit_min:numfit_max], y[numfit_min:numfit_max], facecolor="yellow", alpha=0.4, label='Integrated area') - #plt.plot(x, result.init_fit, 'k--', label='initial fit') - plt.plot(x, result.best_fit, 'k-', label='Best fit') - plt.title('%s \n Gaussian: centre = %9.4f, width = %9.4f, amp = %9.4f \n' - 'Lorentzian: centre, %9.4f, width = %9.4f, amp = %9.4f \n' - 'background: slope = %9.4f, intercept = %9.4f, int_area %9.4f' % (keys, result.params['g_cen'].value, result.params['g_width'].value, - result.params['g_amp'].value, result.params['l_cen'].value, result.params['l_width'].value, - result.params['l_amp'].value, result.params['slope'].value, result.params['intercept'].value, num_int_area)) - - plt.legend(loc='best') - plt.show() d = {} for pars in result.params: d[str(pars)] = (result.params[str(pars)].value, result.params[str(pars)].vary) From 66819afc636a5ef4c8b55d222fe24bb0d150a633 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 11 Sep 2020 15:45:45 +0200 Subject: [PATCH 003/116] updated guard close, deleted plotting --- pyzebra/fit2.py | 180 ++++++++++++++++++++++++++---------------------- 1 file changed, 96 insertions(+), 84 deletions(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 967bb41..720f95b 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -5,7 +5,9 @@ import scipy as sc from scipy import integrate import numpy as np -def fitccl(data, keys, guess, vary, constraints_min, constraints_max, numfit_min=None, numfit_max=None): +def fitccl( + data, keys, guess, vary, constraints_min, constraints_max, numfit_min=None, numfit_max=None +): """Made for fitting of ccl date where 1 peak is expected. Allows for combination of gaussian, lorentzian and linear model combination @@ -29,100 +31,110 @@ def fitccl(data, keys, guess, vary, constraints_min, constraints_max, numfit_min constraints_min = [80, None, 1000, None, None, None, 0, 100] """ - if len(data["Measurements"][str(keys)]["peak_indexes"]) == 1: - x = list(data["Measurements"][str(keys)]["omega"]) - y = list(data["Measurements"][str(keys)]["counts"]) - peak_index = data["Measurements"][str(keys)]["peak_indexes"] - peak_height = data["Measurements"][str(keys)]["peak_heights"] - print('before', constraints_min) - guess[0] = x[int(peak_index)] if guess[0] is None else guess[0] - guess[1] = 0.1 if guess[1] is None else guess[1] - guess[2] = float(peak_height/10) if guess[2]is None else float(guess[2]) - guess[3] = x[int(peak_index)] if guess[3] is None else guess[3] - guess[4] = 2*guess[1] if guess[4] is None else guess[4] - guess[5] = float(peak_height/10) if guess[5] is None else float(guess[5]) - guess[6] = 0 if guess[6] is None else guess[6] - guess[7] = np.median(x) if guess[7] is None else guess[7] - constraints_min[0] = np.min(x) if constraints_min[0] is None else constraints_min[0] - constraints_min[3] = np.min(x) if constraints_min[3] is None else constraints_min[3] - constraints_max[0] = np.max(x) if constraints_max[0] is None else constraints_max[0] - constraints_max[3] = np.max(x) if constraints_max[3] is None else constraints_max[3] - print('key', keys) + if len(data["Measurements"][str(keys)]["peak_indexes"]) != 1: + print("NO PEAK or more than 1 peak") + return - print('after', constraints_min) + x = list(data["Measurements"][str(keys)]["omega"]) + y = list(data["Measurements"][str(keys)]["counts"]) + peak_index = data["Measurements"][str(keys)]["peak_indexes"] + peak_height = data["Measurements"][str(keys)]["peak_heights"] + print("before", constraints_min) + guess[0] = x[int(peak_index)] if guess[0] is None else guess[0] + guess[1] = 0.1 if guess[1] is None else guess[1] + guess[2] = float(peak_height / 10) if guess[2] is None else float(guess[2]) + guess[3] = x[int(peak_index)] if guess[3] is None else guess[3] + guess[4] = 2 * guess[1] if guess[4] is None else guess[4] + guess[5] = float(peak_height / 10) if guess[5] is None else float(guess[5]) + guess[6] = 0 if guess[6] is None else guess[6] + guess[7] = np.median(x) if guess[7] is None else guess[7] + constraints_min[0] = np.min(x) if constraints_min[0] is None else constraints_min[0] + constraints_min[3] = np.min(x) if constraints_min[3] is None else constraints_min[3] + constraints_max[0] = np.max(x) if constraints_max[0] is None else constraints_max[0] + constraints_max[3] = np.max(x) if constraints_max[3] is None else constraints_max[3] + print("key", keys) + print("after", constraints_min) - def find_nearest(array, value): - array = np.asarray(array) - idx = (np.abs(array - value)).argmin() - return idx + def find_nearest(array, value): + array = np.asarray(array) + idx = (np.abs(array - value)).argmin() + return idx - def gaussian(x, g_cen, g_width, g_amp): - """1-d gaussian: gaussian(x, amp, cen, wid)""" - return (g_amp / (np.sqrt(2.0 * np.pi) * g_width)) * np.exp(-(x - g_cen) ** 2 / (2 * g_width ** 2)) + def gaussian(x, g_cen, g_width, g_amp): + """1-d gaussian: gaussian(x, amp, cen, wid)""" + return (g_amp / (np.sqrt(2.0 * np.pi) * g_width)) * np.exp( + -((x - g_cen) ** 2) / (2 * g_width ** 2) + ) - def lorentzian(x, l_cen, l_width, l_amp): - """1d lorentzian""" - return (l_amp / (1 + ((1 * x - l_cen) / l_width) ** 2)) / (np.pi * l_width) + def lorentzian(x, l_cen, l_width, l_amp): + """1d lorentzian""" + return (l_amp / (1 + ((1 * x - l_cen) / l_width) ** 2)) / (np.pi * l_width) - def background(x, slope, intercept): - """background""" - return slope*x + intercept + def background(x, slope, intercept): + """background""" + return slope * x + intercept - mod = Model(gaussian) + Model(lorentzian) + Model(background) - params = Parameters() - params.add_many(('g_cen', x[int(peak_index)], bool(vary[0]), np.min(x), np.max(x), None, None), - ('g_width', guess[1], bool(vary[1]), constraints_min[1], constraints_max[1], None, None), - ('g_amp', guess[2], bool(vary[2]), constraints_min[2], constraints_max[2], None, None), - ('l_cen', guess[3], bool(vary[3]), np.min(x), np.max(x), None, None), - ('l_width', guess[4], bool(vary[4]), constraints_min[4], constraints_max[4], None, None), - ('l_amp', guess[5], bool(vary[5]), constraints_min[5], constraints_max[5], None, None), - ('slope', guess[6], bool(vary[6]), constraints_min[6], constraints_max[6], None, None), - ('intercept', guess[7], bool(vary[7]), constraints_min[7], constraints_max[7], None, None)) + mod = Model(gaussian) + Model(lorentzian) + Model(background) + params = Parameters() + params.add_many( + ("g_cen", x[int(peak_index)], bool(vary[0]), np.min(x), np.max(x), None, None), + ("g_width", guess[1], bool(vary[1]), constraints_min[1], constraints_max[1], None, None), + ("g_amp", guess[2], bool(vary[2]), constraints_min[2], constraints_max[2], None, None), + ("l_cen", guess[3], bool(vary[3]), np.min(x), np.max(x), None, None), + ("l_width", guess[4], bool(vary[4]), constraints_min[4], constraints_max[4], None, None), + ("l_amp", guess[5], bool(vary[5]), constraints_min[5], constraints_max[5], None, None), + ("slope", guess[6], bool(vary[6]), constraints_min[6], constraints_max[6], None, None), + ("intercept", guess[7], bool(vary[7]), constraints_min[7], constraints_max[7], None, None), + ) - result = mod.fit(y, params, x=x) - print('Chi-sqr: ', result.chisqr) + result = mod.fit(y, params, x=x) + print("Chi-sqr: ", result.chisqr) - comps = result.eval_components() - - gauss_3sigmamin = find_nearest(x, result.params['g_cen'].value-3*result.params['g_width'].value) - gauss_3sigmamax = find_nearest(x, result.params['g_cen'].value+3*result.params['g_width'].value) - numfit_min = gauss_3sigmamin if numfit_min is None else find_nearest(x, numfit_min) - numfit_max = gauss_3sigmamax if numfit_max is None else find_nearest(x, numfit_max) - print(numfit_max, numfit_min) - if x[numfit_min] < np.min(x): - numfit_min = gauss_3sigmamin - print('Minimal integration value outside of x range') - elif x[numfit_min] >= x[numfit_max]: - numfit_min = gauss_3sigmamin - print('Minimal integration value higher than maximal') - else: - pass - if x[numfit_max] > np.max(x): - numfit_max = gauss_3sigmamax - print('Maximal integration value outside of x range') - elif x[numfit_max] <= x[numfit_min]: - numfit_max = gauss_3sigmamax - print('Maximal integration value lower than minimal') - else: - pass - print(result.params['g_width'].value) - print(result.params['g_cen'].value) - num_int_area = simps(y[numfit_min:numfit_max], x[numfit_min:numfit_max]) - num_int_bacground = integrate.quad(background, numfit_min, numfit_max, args=(result.params['slope'].value,result.params['intercept'].value)) - d = {} - for pars in result.params: - d[str(pars)] = (result.params[str(pars)].value, result.params[str(pars)].vary) - - d["int_area"] = num_int_area - d["int_background"] = num_int_bacground - d["full_report"] = result.fit_report() - data["Measurements"][str(keys)]["fit"] = d - - return data + comps = result.eval_components() + gauss_3sigmamin = find_nearest( + x, result.params["g_cen"].value - 3 * result.params["g_width"].value + ) + gauss_3sigmamax = find_nearest( + x, result.params["g_cen"].value + 3 * result.params["g_width"].value + ) + numfit_min = gauss_3sigmamin if numfit_min is None else find_nearest(x, numfit_min) + numfit_max = gauss_3sigmamax if numfit_max is None else find_nearest(x, numfit_max) + print(numfit_max, numfit_min) + if x[numfit_min] < np.min(x): + numfit_min = gauss_3sigmamin + print("Minimal integration value outside of x range") + elif x[numfit_min] >= x[numfit_max]: + numfit_min = gauss_3sigmamin + print("Minimal integration value higher than maximal") else: - return print('NO PEAK or more than 1 peak') + pass + if x[numfit_max] > np.max(x): + numfit_max = gauss_3sigmamax + print("Maximal integration value outside of x range") + elif x[numfit_max] <= x[numfit_min]: + numfit_max = gauss_3sigmamax + print("Maximal integration value lower than minimal") + else: + pass + print(result.params["g_width"].value) + print(result.params["g_cen"].value) + num_int_area = simps(y[numfit_min:numfit_max], x[numfit_min:numfit_max]) + num_int_bacground = integrate.quad( + background, + numfit_min, + numfit_max, + args=(result.params["slope"].value, result.params["intercept"].value), + ) + d = {} + for pars in result.params: + d[str(pars)] = (result.params[str(pars)].value, result.params[str(pars)].vary) + d["int_area"] = num_int_area + d["int_background"] = num_int_bacground + d["full_report"] = result.fit_report() + data["Measurements"][str(keys)]["fit"] = d + return data From 2788c77fa19397817c1c1f735fb002533a98ac64 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 11 Sep 2020 17:58:57 +0200 Subject: [PATCH 004/116] Remove unused imports --- pyzebra/fit2.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 720f95b..36fc1ae 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -1,9 +1,8 @@ -from lmfit import minimize, Parameters, Model -from lmfit.models import LinearModel, LorentzianModel, GaussianModel -from scipy.integrate import simps -import scipy as sc -from scipy import integrate import numpy as np +from lmfit import Model, Parameters +from scipy import integrate +from scipy.integrate import simps + def fitccl( data, keys, guess, vary, constraints_min, constraints_max, numfit_min=None, numfit_max=None From b157c2a3ae17596b0cfd2c04217ea39fcc29b3ef Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Mon, 14 Sep 2020 13:43:56 +0200 Subject: [PATCH 005/116] Update fit2.py corrected value range for numerical fit, added export_fit parameter, which should in future decide if exported value is fitted or integrated one. --- pyzebra/fit2.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 36fc1ae..962cfc7 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -122,15 +122,16 @@ def fitccl( num_int_area = simps(y[numfit_min:numfit_max], x[numfit_min:numfit_max]) num_int_bacground = integrate.quad( background, - numfit_min, - numfit_max, + x[numfit_min], + x[numfit_max], args=(result.params["slope"].value, result.params["intercept"].value), ) d = {} for pars in result.params: d[str(pars)] = (result.params[str(pars)].value, result.params[str(pars)].vary) - + + d["export_fit"] = False d["int_area"] = num_int_area d["int_background"] = num_int_bacground d["full_report"] = result.fit_report() From aa9b0c99ae042310e50b4789ab541cce9b676945 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Mon, 14 Sep 2020 14:34:26 +0200 Subject: [PATCH 006/116] Update fit2.py While function to find different indexes if the peak is very thin and sigma is much smaller than step, added the find_nearest which was missing for some reason. --- pyzebra/fit2.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 962cfc7..ac499e3 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -4,6 +4,12 @@ from scipy import integrate from scipy.integrate import simps +def find_nearest(array, value): + array = np.asarray(array) + idx = (np.abs(array - value)).argmin() + return idx + + def fitccl( data, keys, guess, vary, constraints_min, constraints_max, numfit_min=None, numfit_max=None ): @@ -100,7 +106,17 @@ def fitccl( ) numfit_min = gauss_3sigmamin if numfit_min is None else find_nearest(x, numfit_min) numfit_max = gauss_3sigmamax if numfit_max is None else find_nearest(x, numfit_max) - print(numfit_max, numfit_min) + it = -1 + + while numfit_max == numfit_min: + it = it + 1 + numfit_min = find_nearest( + x, result.params["g_cen"].value - 3 * (1 + it / 10) * result.params["g_width"].value + ) + numfit_max = find_nearest( + x, result.params["g_cen"].value + 3 * (1 + it / 10) * result.params["g_width"].value + ) + if x[numfit_min] < np.min(x): numfit_min = gauss_3sigmamin print("Minimal integration value outside of x range") @@ -130,8 +146,8 @@ def fitccl( d = {} for pars in result.params: d[str(pars)] = (result.params[str(pars)].value, result.params[str(pars)].vary) - - d["export_fit"] = False + + d["export_fit"] = False d["int_area"] = num_int_area d["int_background"] = num_int_bacground d["full_report"] = result.fit_report() From 5ad3ccc3a2596a705e8296d532681fb3635f77fd Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Thu, 10 Sep 2020 15:17:55 +0200 Subject: [PATCH 007/116] Function to export *.comm files --- pyzebra/comm_export.py | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 pyzebra/comm_export.py diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py new file mode 100644 index 0000000..695a5c0 --- /dev/null +++ b/pyzebra/comm_export.py @@ -0,0 +1,43 @@ +path = 'C:\\Users\\Jakub\\Desktop\\' +name = 'test' +extension = '.comm' + + +def fill_string(string, total_lenght): + """fills string to tatal_length with whitespaces + :arg string - string to fill + :arg total_lenght - length of the final string + :return string of the lenght of total_lenght + """ + white_spaces = ' '*total_lenght + return_string = white_spaces[:-len(str(string))] + str(string) + return return_string + + +def export_comm(data, name): + """exports data in the *.comm format + :arg data - data to export, is dict after peak fitting + arg: name - name of the exported file + """ + out_file = open(str(path+name+extension), "w") + for keys in data["Measurements"]: + try: + meas_number_str = fill_string(keys[1:], 6) + h_str = fill_string(int(data["Measurements"][str(keys)]['h_index']), 4) + k_str = fill_string(int(data["Measurements"][str(keys)]['k_index']), 4) + l_str = fill_string(int(data["Measurements"][str(keys)]['l_index']), 4) + int_str = fill_string("{:10.2f}".format(data["Measurements"][str(keys)]['fit']['g_amp'][0]), 10) + sigma_str = fill_string("{:10.2f}".format(data["Measurements"][str(keys)]['fit']['g_width'][0]), 10) + twotheta_str = fill_string("{:8.2f}".format(data["Measurements"][str(keys)]["twotheta_angle"]), 8) + omega_str = fill_string("{:8.2f}".format(data["Measurements"][str(keys)]["omega_angle"]), 8) + chi_str = fill_string("{:8.2f}".format(data["Measurements"][str(keys)]["chi_angle"]), 8) + phi_str = fill_string("{:8.2f}".format(data["Measurements"][str(keys)]["phi_angle"]), 8) + line = meas_number_str + h_str + l_str + k_str + int_str + sigma_str + twotheta_str + omega_str + chi_str + phi_str + "\n" + out_file.write(line) + print('Measurement written:', keys) + + except KeyError: + print('Measurement skipped - no fit value for:', keys) + + out_file.close() + From a24982fa0bed55078e4952e7d3a588d96b3c4da9 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 11 Sep 2020 14:55:41 +0200 Subject: [PATCH 008/116] Update comm_export.py --- pyzebra/comm_export.py | 142 ++++++++++++++++++++++++++++++++++------- 1 file changed, 118 insertions(+), 24 deletions(-) diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py index 695a5c0..654d6cb 100644 --- a/pyzebra/comm_export.py +++ b/pyzebra/comm_export.py @@ -1,6 +1,5 @@ -path = 'C:\\Users\\Jakub\\Desktop\\' -name = 'test' -extension = '.comm' +path = "C:\\Users\\Jakub\\Desktop\\" +name = "test" def fill_string(string, total_lenght): @@ -9,8 +8,8 @@ def fill_string(string, total_lenght): :arg total_lenght - length of the final string :return string of the lenght of total_lenght """ - white_spaces = ' '*total_lenght - return_string = white_spaces[:-len(str(string))] + str(string) + white_spaces = " " * total_lenght + return_string = white_spaces[: -len(str(string))] + str(string) return return_string @@ -19,25 +18,120 @@ def export_comm(data, name): :arg data - data to export, is dict after peak fitting arg: name - name of the exported file """ - out_file = open(str(path+name+extension), "w") - for keys in data["Measurements"]: - try: - meas_number_str = fill_string(keys[1:], 6) - h_str = fill_string(int(data["Measurements"][str(keys)]['h_index']), 4) - k_str = fill_string(int(data["Measurements"][str(keys)]['k_index']), 4) - l_str = fill_string(int(data["Measurements"][str(keys)]['l_index']), 4) - int_str = fill_string("{:10.2f}".format(data["Measurements"][str(keys)]['fit']['g_amp'][0]), 10) - sigma_str = fill_string("{:10.2f}".format(data["Measurements"][str(keys)]['fit']['g_width'][0]), 10) - twotheta_str = fill_string("{:8.2f}".format(data["Measurements"][str(keys)]["twotheta_angle"]), 8) - omega_str = fill_string("{:8.2f}".format(data["Measurements"][str(keys)]["omega_angle"]), 8) - chi_str = fill_string("{:8.2f}".format(data["Measurements"][str(keys)]["chi_angle"]), 8) - phi_str = fill_string("{:8.2f}".format(data["Measurements"][str(keys)]["phi_angle"]), 8) - line = meas_number_str + h_str + l_str + k_str + int_str + sigma_str + twotheta_str + omega_str + chi_str + phi_str + "\n" - out_file.write(line) - print('Measurement written:', keys) - except KeyError: - print('Measurement skipped - no fit value for:', keys) + if data["meta"]["indices"] == "hkl": + extension = ".comm" + with open(str(path + name + extension), "w") as out_file: + for keys in data["Measurements"]: + try: + meas_number_str = fill_string(keys[1:], 6) + h_str = fill_string(int(data["Measurements"][str(keys)]["h_index"]), 4) + k_str = fill_string(int(data["Measurements"][str(keys)]["k_index"]), 4) + l_str = fill_string(int(data["Measurements"][str(keys)]["l_index"]), 4) + int_str = fill_string( + "{:10.2f}".format(data["Measurements"][str(keys)]["fit"]["g_amp"][0]), 10 + ) + sigma_str = fill_string( + "{:10.2f}".format(data["Measurements"][str(keys)]["fit"]["g_width"][0]), 10 + ) + if data["meta"]["zebra_mode"] == 'bi': + twotheta_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["twotheta_angle"]), 8 + ) + omega_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["omega_angle"]), 8 + ) + chi_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["chi_angle"]), 8 + ) + phi_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["phi_angle"]), 8 + ) + elif data["meta"]["zebra_mode"] == 'nb': + twotheta_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["gamma_angle"]), 8 + ) + omega_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["omega_angle"]), 8 + ) + chi_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["nu_angle"]), 8 + ) + phi_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["unkwn_angle"]), 8 + ) - out_file.close() + line = ( + meas_number_str + + h_str + + l_str + + k_str + + int_str + + sigma_str + + twotheta_str + + omega_str + + chi_str + + phi_str + + "\n" + ) + out_file.write(line) + except KeyError: + print("Measurement skipped - no fit value for:", keys) + elif data["meta"]["indices"] == "real": + extension = ".incomm" + with open(str(path + name + extension), "w") as out_file: + for keys in data["Measurements"]: + try: + meas_number_str = fill_string(keys[1:], 4) + h_str = fill_string(int(data["Measurements"][str(keys)]["h_index"]), 6) + k_str = fill_string(int(data["Measurements"][str(keys)]["k_index"]), 6) + l_str = fill_string(int(data["Measurements"][str(keys)]["l_index"]), 6) + int_str = fill_string( + "{:10.2f}".format(data["Measurements"][str(keys)]["fit"]["g_amp"][0]), 10 + ) + sigma_str = fill_string( + "{:10.2f}".format(data["Measurements"][str(keys)]["fit"]["g_width"][0]), 10 + ) + if data["meta"]["zebra_mode"] == 'bi': + twotheta_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["twotheta_angle"]), 8 + ) + omega_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["omega_angle"]), 8 + ) + chi_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["chi_angle"]), 8 + ) + phi_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["phi_angle"]), 8 + ) + elif data["meta"]["zebra_mode"] == 'nb': + twotheta_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["gamma_angle"]), 8 + ) + omega_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["omega_angle"]), 8 + ) + chi_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["nu_angle"]), 8 + ) + phi_str = fill_string( + "{:8.2f}".format(data["Measurements"][str(keys)]["unkwn_angle"]), 8 + ) + line = ( + meas_number_str + + h_str + + l_str + + k_str + + int_str + + sigma_str + + twotheta_str + + omega_str + + chi_str + + phi_str + + "\n" + ) + out_file.write(line) + except KeyError: + print("Measurement skipped - no fit value for:", keys) From c6636ef212cc77777d5076f60fdc22feb4ed083f Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 11 Sep 2020 15:38:54 +0200 Subject: [PATCH 009/116] removed strings, added argument --- pyzebra/comm_export.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py index 654d6cb..29110cd 100644 --- a/pyzebra/comm_export.py +++ b/pyzebra/comm_export.py @@ -1,6 +1,3 @@ -path = "C:\\Users\\Jakub\\Desktop\\" -name = "test" - def fill_string(string, total_lenght): """fills string to tatal_length with whitespaces @@ -13,15 +10,15 @@ def fill_string(string, total_lenght): return return_string -def export_comm(data, name): +def export_comm(data, path): """exports data in the *.comm format + :param path: path to file + name :arg data - data to export, is dict after peak fitting - arg: name - name of the exported file """ if data["meta"]["indices"] == "hkl": extension = ".comm" - with open(str(path + name + extension), "w") as out_file: + with open(str(path + extension), "w") as out_file: for keys in data["Measurements"]: try: meas_number_str = fill_string(keys[1:], 6) @@ -80,7 +77,7 @@ def export_comm(data, name): elif data["meta"]["indices"] == "real": extension = ".incomm" - with open(str(path + name + extension), "w") as out_file: + with open(str(path + extension), "w") as out_file: for keys in data["Measurements"]: try: meas_number_str = fill_string(keys[1:], 4) From 870fb745ba8a7657d10be13d9addeff541bc40fb Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Mon, 14 Sep 2020 15:22:29 +0200 Subject: [PATCH 010/116] Update comm_export.py reworked function to export data. Added lorenntz to decide if Lorentz correction should be done and also export_fit to decide whether fitter or itegrated area is to be exported (added also to fit2.py) --- pyzebra/comm_export.py | 188 +++++++++++++++-------------------------- 1 file changed, 67 insertions(+), 121 deletions(-) diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py index 29110cd..eac4c8b 100644 --- a/pyzebra/comm_export.py +++ b/pyzebra/comm_export.py @@ -1,134 +1,80 @@ - -def fill_string(string, total_lenght): - """fills string to tatal_length with whitespaces - :arg string - string to fill - :arg total_lenght - length of the final string - :return string of the lenght of total_lenght - """ - white_spaces = " " * total_lenght - return_string = white_spaces[: -len(str(string))] + str(string) - return return_string +import numpy as np -def export_comm(data, path): +def correction(value, lorentz=True, zebra_mode="--", ang1=0, ang2=0): + if lorentz is False: + return value + else: + if zebra_mode == "bi": + corr_value = np.abs(value * np.sin(ang1)) + return corr_value + elif zebra_mode == "nb": + corr_value = np.abs(value * np.sin(ang1) * np.cos(ang2)) + return corr_value + + +def export_comm(data, path, lorentz=False): """exports data in the *.comm format + :param lorentz: perform Lorentz correction :param path: path to file + name :arg data - data to export, is dict after peak fitting + """ + align = ">" if data["meta"]["indices"] == "hkl": extension = ".comm" - with open(str(path + extension), "w") as out_file: - for keys in data["Measurements"]: - try: - meas_number_str = fill_string(keys[1:], 6) - h_str = fill_string(int(data["Measurements"][str(keys)]["h_index"]), 4) - k_str = fill_string(int(data["Measurements"][str(keys)]["k_index"]), 4) - l_str = fill_string(int(data["Measurements"][str(keys)]["l_index"]), 4) - int_str = fill_string( - "{:10.2f}".format(data["Measurements"][str(keys)]["fit"]["g_amp"][0]), 10 - ) - sigma_str = fill_string( - "{:10.2f}".format(data["Measurements"][str(keys)]["fit"]["g_width"][0]), 10 - ) - if data["meta"]["zebra_mode"] == 'bi': - twotheta_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["twotheta_angle"]), 8 - ) - omega_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["omega_angle"]), 8 - ) - chi_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["chi_angle"]), 8 - ) - phi_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["phi_angle"]), 8 - ) - elif data["meta"]["zebra_mode"] == 'nb': - twotheta_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["gamma_angle"]), 8 - ) - omega_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["omega_angle"]), 8 - ) - chi_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["nu_angle"]), 8 - ) - phi_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["unkwn_angle"]), 8 - ) - - line = ( - meas_number_str - + h_str - + l_str - + k_str - + int_str - + sigma_str - + twotheta_str - + omega_str - + chi_str - + phi_str - + "\n" - ) - out_file.write(line) - except KeyError: - print("Measurement skipped - no fit value for:", keys) - + padding = [6, 4, 10, 8] elif data["meta"]["indices"] == "real": extension = ".incomm" - with open(str(path + extension), "w") as out_file: - for keys in data["Measurements"]: - try: - meas_number_str = fill_string(keys[1:], 4) - h_str = fill_string(int(data["Measurements"][str(keys)]["h_index"]), 6) - k_str = fill_string(int(data["Measurements"][str(keys)]["k_index"]), 6) - l_str = fill_string(int(data["Measurements"][str(keys)]["l_index"]), 6) - int_str = fill_string( - "{:10.2f}".format(data["Measurements"][str(keys)]["fit"]["g_amp"][0]), 10 + padding = [4, 6, 10, 8] + + with open(str(path + extension), "w") as out_file: + for keys in data["Measurements"]: + print(keys) + try: + meas_number_str = f"{keys[1:]:{align}{padding[0]}}" + h_str = f'{int(data["Measurements"][str(keys)]["h_index"]):{padding[1]}}' + k_str = f'{int(data["Measurements"][str(keys)]["k_index"]):{padding[1]}}' + l_str = f'{int(data["Measurements"][str(keys)]["l_index"]):{padding[1]}}' + if data["Measurements"][str(keys)]["fit"]["export_fit"] is True: + area = float(data["Measurements"][str(keys)]["fit"]["g_amp"][0]) + float( + data["Measurements"][str(keys)]["fit"]["l_amp"][0] ) - sigma_str = fill_string( - "{:10.2f}".format(data["Measurements"][str(keys)]["fit"]["g_width"][0]), 10 + else: + area = float(data["Measurements"][str(keys)]["fit"]["int_area"]) - float( + data["Measurements"][str(keys)]["fit"]["int_background"][0] ) - if data["meta"]["zebra_mode"] == 'bi': - twotheta_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["twotheta_angle"]), 8 - ) - omega_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["omega_angle"]), 8 - ) - chi_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["chi_angle"]), 8 - ) - phi_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["phi_angle"]), 8 - ) - elif data["meta"]["zebra_mode"] == 'nb': - twotheta_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["gamma_angle"]), 8 - ) - omega_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["omega_angle"]), 8 - ) - chi_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["nu_angle"]), 8 - ) - phi_str = fill_string( - "{:8.2f}".format(data["Measurements"][str(keys)]["unkwn_angle"]), 8 - ) - line = ( - meas_number_str - + h_str - + l_str - + k_str - + int_str - + sigma_str - + twotheta_str - + omega_str - + chi_str - + phi_str - + "\n" - ) - out_file.write(line) - except KeyError: - print("Measurement skipped - no fit value for:", keys) + + if data["meta"]["zebra_mode"] == "bi": + int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], data["Measurements"][str(keys)]["twotheta_angle"])):{align}{padding[2]}}' + angle_str1 = f'{data["Measurements"][str(keys)]["twotheta_angle"]:{padding[3]}}' + angle_str2 = f'{data["Measurements"][str(keys)]["omega_angle"]:{padding[3]}}' + angle_str3 = f'{data["Measurements"][str(keys)]["chi_angle"]:{padding[3]}}' + angle_str4 = f'{data["Measurements"][str(keys)]["phi_angle"]:{padding[3]}}' + elif data["meta"]["zebra_mode"] == "nb": + int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], data["Measurements"][str(keys)]["gamma_angle"],data["Measurements"][str(keys)]["nu_angle"])):{align}{padding[2]}}' + angle_str1 = f'{data["Measurements"][str(keys)]["gamma_angle"]:{padding[3]}}' + angle_str2 = f'{data["Measurements"][str(keys)]["omega_angle"]:{padding[3]}}' + angle_str3 = f'{data["Measurements"][str(keys)]["nu_angle"]:{padding[3]}}' + angle_str4 = f'{data["Measurements"][str(keys)]["unkwn_angle"]:{padding[3]}}' + + sigma_str = f'{"{:8.2f}".format(float(data["Measurements"][str(keys)]["fit"]["g_width"][0])):{align}{padding[2]}}' + line = ( + meas_number_str + + h_str + + l_str + + k_str + + int_str + + sigma_str + + angle_str1 + + angle_str2 + + angle_str3 + + angle_str4 + + "\n" + ) + out_file.write(line) + + except KeyError: + print("Measurement skipped - no fit value for:", keys) + From 96f1b3610949c73ff4199578f5938350a807f1eb Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Thu, 10 Sep 2020 15:09:19 +0200 Subject: [PATCH 011/116] Add files via upload Try, except deleted. Added one if/elif loop to tell apart zebra mode (bi, nb) and different angles saved --- pyzebra/load_1D.py | 111 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 pyzebra/load_1D.py diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py new file mode 100644 index 0000000..a3520a2 --- /dev/null +++ b/pyzebra/load_1D.py @@ -0,0 +1,111 @@ +import re +import numpy as np + + +def load_1D(filepath): + """ + Loads *.ccl or *.dat file (Distinguishes them based on last 3 chars in string of filepath + to add more variables to read, extend the elif list + the file must include '#data' and number of points in right place to work properly + + :arg filepath + :returns det_variables + - dictionary of all detector/scan variables and dictinionary for every measurement. + Names of these dictionaries are M + measurement number. They include HKL indeces, angles, monitors, + stepsize and array of counts + """ + + det_variables = {"file_type": str(filepath)[-3:], "meta": {}} + meta_vars_str = ('instrument', 'title', 'sample', 'user', 'ProposalID', 'original_filename', 'date', + 'zebra_mode','proposal', 'proposal_user', 'proposal_title', 'proposal_email') + meta_vars_float = ('mf', '2-theta', 'chi', 'phi', 'nu', 'temp', 'wavelenght', 'a', 'b', 'c', 'alpha', 'beta', + 'gamma', 'cex1', 'cex2', 'mexz', 'moml', 'mcvl', 'momu', 'mcvu', 'detectorDistance', 'snv', + 'snh', 'snvm', 'snhm', 's1vt', 's1vb', 's1hr', 's1hl', 's2vt', 's2vb', 's2hr', 's2hl') + meta_ub_matrix = ('ub1j', 'ub2j', 'ub3j') + with open(filepath, 'r') as infile: + for line in infile: + det_variables["Measurements"] = {} + if '=' in line: + variable, value = line.split('=') + variable = variable.strip() + try: + if variable in meta_vars_float: + det_variables["meta"][variable] = float(value) + elif variable in meta_vars_str: + det_variables["meta"][variable] = str(value)[:-1].strip() + elif variable in meta_ub_matrix: + det_variables["meta"][variable] = re.findall(r"[-+]?\d*\.\d+|\d+", str(value)) + except ValueError as error: + print('Some values are not in expected format (str or float), error:', str(error)) + elif '#data' in line: + if det_variables["file_type"] == 'ccl': + data = infile.readlines() + position = - 1 + for lines in data: + position = position + 1 + if bool(re.match('(\s\s\s\d)', lines[0:4])) == True or bool( + re.match('(\s\s\d\d)', lines[0:4])) == True or bool( + re.match('(\s\d\d\d)', lines[0:4])) == True or bool( + re.match('(\d\d\d\d)', lines[0:4])) == True: + counts = [] + measurement_number = int(lines.split()[0]) + d = {} + d["h_index"] = float(lines.split()[1]) + d["k_index"] = float(lines.split()[2]) + d["l_index"] = float(lines.split()[3]) + if det_variables["meta"]["zebra_mode"] == 'bi': + d["twotheta_angle"] = float(lines.split()[4]) # gamma + d["omega_angle"] = float(lines.split()[5]) # omega + d["chi_angle"] = float(lines.split()[6]) # nu + d["phi_angle"] = float(lines.split()[7]) # doesnt matter + elif det_variables["meta"]["zebra_mode"] == 'nb': + d["gamma_angle"] = float(lines.split()[4]) # gamma + d["omega_angle"] = float(lines.split()[5]) # omega + d["nu_angle"] = float(lines.split()[6]) # nu + + next_line = data[position + 1] + d["number_of_measurements"] = int(next_line.split()[0]) + d["angle_step"] = float(next_line.split()[1]) + d["monitor"] = float(next_line.split()[2]) + d["unkwn1"] = float(next_line.split()[3]) + d["unkwn2"] = float(next_line.split()[4]) + d["date"] = str(next_line.split()[5]) + d["time"] = str(next_line.split()[6]) + d["scan_type"] = str(next_line.split()[7]) + for i in range( + int(int(next_line.split()[0]) / 10) + (int(next_line.split()[0]) % 10 > 0)): + fileline = data[position + 2 + i].split() + numbers = [int(w) for w in fileline] + counts = counts + numbers + d["omega"] = np.linspace( + float(lines.split()[5]) - (int(next_line.split()[0]) / 2) * float( + next_line.split()[1]), + float(lines.split()[5]) + (int(next_line.split()[0]) / 2) * float( + next_line.split()[1]), int(next_line.split()[0])) + d["counts"] = counts + det_variables["Measurements"][str('M' + str(measurement_number))] = d + elif det_variables["file_type"] == 'dat': + data = infile.readlines() + num_of_points = int(data[1].split()[0]) + omega = [] + counts = [] + monitor1 = [] + monitor2 = [] + monitor3 = [] + time = [] + for position in range(num_of_points): + omega.append(float(data[position + 3].split()[1])) + counts.append(float(data[position + 3].split()[2])) + monitor1.append(float(data[position + 3].split()[3])) + monitor2.append(float(data[position + 3].split()[4])) + monitor3.append(float(data[position + 3].split()[5])) + time.append(float(data[position + 3].split()[6])) + det_variables["Measurements"]["omega"] = omega + det_variables["Measurements"]["counts"] = counts + det_variables["Measurements"]["Monitor1"] = monitor1 + det_variables["Measurements"]["Monitor2"] = monitor2 + det_variables["Measurements"]["Monitor3"] = monitor3 + det_variables["Measurements"]["time"] = time + else: + print('Unknown file extention') + return det_variables From ddebcc305ffd0a96cb2877c9e8fe9b5a03c3b662 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 11 Sep 2020 11:35:36 +0200 Subject: [PATCH 012/116] Extract constants --- pyzebra/load_1D.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index a3520a2..7eede2c 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -1,6 +1,12 @@ import re import numpy as np +META_VARS_STR = ('instrument', 'title', 'sample', 'user', 'ProposalID', 'original_filename', 'date', + 'zebra_mode','proposal', 'proposal_user', 'proposal_title', 'proposal_email') +META_VARS_FLOAT = ('mf', '2-theta', 'chi', 'phi', 'nu', 'temp', 'wavelenght', 'a', 'b', 'c', 'alpha', 'beta', + 'gamma', 'cex1', 'cex2', 'mexz', 'moml', 'mcvl', 'momu', 'mcvu', 'detectorDistance', 'snv', + 'snh', 'snvm', 'snhm', 's1vt', 's1vb', 's1hr', 's1hl', 's2vt', 's2vb', 's2hr', 's2hl') +META_UB_MATRIX = ('ub1j', 'ub2j', 'ub3j') def load_1D(filepath): """ @@ -16,12 +22,6 @@ def load_1D(filepath): """ det_variables = {"file_type": str(filepath)[-3:], "meta": {}} - meta_vars_str = ('instrument', 'title', 'sample', 'user', 'ProposalID', 'original_filename', 'date', - 'zebra_mode','proposal', 'proposal_user', 'proposal_title', 'proposal_email') - meta_vars_float = ('mf', '2-theta', 'chi', 'phi', 'nu', 'temp', 'wavelenght', 'a', 'b', 'c', 'alpha', 'beta', - 'gamma', 'cex1', 'cex2', 'mexz', 'moml', 'mcvl', 'momu', 'mcvu', 'detectorDistance', 'snv', - 'snh', 'snvm', 'snhm', 's1vt', 's1vb', 's1hr', 's1hl', 's2vt', 's2vb', 's2hr', 's2hl') - meta_ub_matrix = ('ub1j', 'ub2j', 'ub3j') with open(filepath, 'r') as infile: for line in infile: det_variables["Measurements"] = {} @@ -29,11 +29,11 @@ def load_1D(filepath): variable, value = line.split('=') variable = variable.strip() try: - if variable in meta_vars_float: + if variable in META_VARS_FLOAT: det_variables["meta"][variable] = float(value) - elif variable in meta_vars_str: + elif variable in META_VARS_STR: det_variables["meta"][variable] = str(value)[:-1].strip() - elif variable in meta_ub_matrix: + elif variable in META_UB_MATRIX: det_variables["meta"][variable] = re.findall(r"[-+]?\d*\.\d+|\d+", str(value)) except ValueError as error: print('Some values are not in expected format (str or float), error:', str(error)) From 661014ad73ac0438383a149ff58d8d6777763cab Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 11 Sep 2020 12:00:50 +0200 Subject: [PATCH 013/116] Apply black formatter --- pyzebra/load_1D.py | 133 ++++++++++++++++++++++++++++++++------------- 1 file changed, 95 insertions(+), 38 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 7eede2c..d2d6e01 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -1,12 +1,57 @@ import re import numpy as np -META_VARS_STR = ('instrument', 'title', 'sample', 'user', 'ProposalID', 'original_filename', 'date', - 'zebra_mode','proposal', 'proposal_user', 'proposal_title', 'proposal_email') -META_VARS_FLOAT = ('mf', '2-theta', 'chi', 'phi', 'nu', 'temp', 'wavelenght', 'a', 'b', 'c', 'alpha', 'beta', - 'gamma', 'cex1', 'cex2', 'mexz', 'moml', 'mcvl', 'momu', 'mcvu', 'detectorDistance', 'snv', - 'snh', 'snvm', 'snhm', 's1vt', 's1vb', 's1hr', 's1hl', 's2vt', 's2vb', 's2hr', 's2hl') -META_UB_MATRIX = ('ub1j', 'ub2j', 'ub3j') +META_VARS_STR = ( + "instrument", + "title", + "sample", + "user", + "ProposalID", + "original_filename", + "date", + "zebra_mode", + "proposal", + "proposal_user", + "proposal_title", + "proposal_email", +) +META_VARS_FLOAT = ( + "mf", + "2-theta", + "chi", + "phi", + "nu", + "temp", + "wavelenght", + "a", + "b", + "c", + "alpha", + "beta", + "gamma", + "cex1", + "cex2", + "mexz", + "moml", + "mcvl", + "momu", + "mcvu", + "detectorDistance", + "snv", + "snh", + "snvm", + "snhm", + "s1vt", + "s1vb", + "s1hr", + "s1hl", + "s2vt", + "s2vb", + "s2hr", + "s2hl", +) +META_UB_MATRIX = ("ub1j", "ub2j", "ub3j") + def load_1D(filepath): """ @@ -17,16 +62,15 @@ def load_1D(filepath): :arg filepath :returns det_variables - dictionary of all detector/scan variables and dictinionary for every measurement. - Names of these dictionaries are M + measurement number. They include HKL indeces, angles, monitors, - stepsize and array of counts + Names of these dictionaries are M + measurement number. They include HKL indeces, angles, + monitors, stepsize and array of counts """ - det_variables = {"file_type": str(filepath)[-3:], "meta": {}} - with open(filepath, 'r') as infile: + with open(filepath, "r") as infile: for line in infile: det_variables["Measurements"] = {} - if '=' in line: - variable, value = line.split('=') + if "=" in line: + variable, value = line.split("=") variable = variable.strip() try: if variable in META_VARS_FLOAT: @@ -34,34 +78,41 @@ def load_1D(filepath): elif variable in META_VARS_STR: det_variables["meta"][variable] = str(value)[:-1].strip() elif variable in META_UB_MATRIX: - det_variables["meta"][variable] = re.findall(r"[-+]?\d*\.\d+|\d+", str(value)) + det_variables["meta"][variable] = re.findall( + r"[-+]?\d*\.\d+|\d+", str(value) + ) except ValueError as error: - print('Some values are not in expected format (str or float), error:', str(error)) - elif '#data' in line: - if det_variables["file_type"] == 'ccl': + print( + "Some values are not in expected format (str or float), error:", str(error) + ) + + elif "#data" in line: + if det_variables["file_type"] == "ccl": data = infile.readlines() - position = - 1 + position = -1 for lines in data: position = position + 1 - if bool(re.match('(\s\s\s\d)', lines[0:4])) == True or bool( - re.match('(\s\s\d\d)', lines[0:4])) == True or bool( - re.match('(\s\d\d\d)', lines[0:4])) == True or bool( - re.match('(\d\d\d\d)', lines[0:4])) == True: + if ( + bool(re.match("(\s\s\s\d)", lines[0:4])) == True + or bool(re.match("(\s\s\d\d)", lines[0:4])) == True + or bool(re.match("(\s\d\d\d)", lines[0:4])) == True + or bool(re.match("(\d\d\d\d)", lines[0:4])) == True + ): counts = [] measurement_number = int(lines.split()[0]) d = {} d["h_index"] = float(lines.split()[1]) d["k_index"] = float(lines.split()[2]) d["l_index"] = float(lines.split()[3]) - if det_variables["meta"]["zebra_mode"] == 'bi': - d["twotheta_angle"] = float(lines.split()[4]) # gamma - d["omega_angle"] = float(lines.split()[5]) # omega - d["chi_angle"] = float(lines.split()[6]) # nu - d["phi_angle"] = float(lines.split()[7]) # doesnt matter - elif det_variables["meta"]["zebra_mode"] == 'nb': - d["gamma_angle"] = float(lines.split()[4]) # gamma - d["omega_angle"] = float(lines.split()[5]) # omega - d["nu_angle"] = float(lines.split()[6]) # nu + if det_variables["meta"]["zebra_mode"] == "bi": + d["twotheta_angle"] = float(lines.split()[4]) # gamma + d["omega_angle"] = float(lines.split()[5]) # omega + d["chi_angle"] = float(lines.split()[6]) # nu + d["phi_angle"] = float(lines.split()[7]) # doesnt matter + elif det_variables["meta"]["zebra_mode"] == "nb": + d["gamma_angle"] = float(lines.split()[4]) # gamma + d["omega_angle"] = float(lines.split()[5]) # omega + d["nu_angle"] = float(lines.split()[6]) # nu next_line = data[position + 1] d["number_of_measurements"] = int(next_line.split()[0]) @@ -73,18 +124,23 @@ def load_1D(filepath): d["time"] = str(next_line.split()[6]) d["scan_type"] = str(next_line.split()[7]) for i in range( - int(int(next_line.split()[0]) / 10) + (int(next_line.split()[0]) % 10 > 0)): + int(int(next_line.split()[0]) / 10) + + (int(next_line.split()[0]) % 10 > 0) + ): fileline = data[position + 2 + i].split() numbers = [int(w) for w in fileline] counts = counts + numbers d["omega"] = np.linspace( - float(lines.split()[5]) - (int(next_line.split()[0]) / 2) * float( - next_line.split()[1]), - float(lines.split()[5]) + (int(next_line.split()[0]) / 2) * float( - next_line.split()[1]), int(next_line.split()[0])) + float(lines.split()[5]) + - (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), + float(lines.split()[5]) + + (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), + int(next_line.split()[0]), + ) d["counts"] = counts - det_variables["Measurements"][str('M' + str(measurement_number))] = d - elif det_variables["file_type"] == 'dat': + det_variables["Measurements"][str("M" + str(measurement_number))] = d + + elif det_variables["file_type"] == "dat": data = infile.readlines() num_of_points = int(data[1].split()[0]) omega = [] @@ -107,5 +163,6 @@ def load_1D(filepath): det_variables["Measurements"]["Monitor3"] = monitor3 det_variables["Measurements"]["time"] = time else: - print('Unknown file extention') + print("Unknown file extention") + return det_variables From 74151dd66795d0c230fbb7acfd775922c8aaeba4 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 11 Sep 2020 12:12:27 +0200 Subject: [PATCH 014/116] Stop in case of malformed metadata section Remove exception hadling because if there is an error in reading, we shouldn't return partially read data, which will cause issues later. --- pyzebra/load_1D.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index d2d6e01..e3aaf05 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -72,19 +72,12 @@ def load_1D(filepath): if "=" in line: variable, value = line.split("=") variable = variable.strip() - try: - if variable in META_VARS_FLOAT: - det_variables["meta"][variable] = float(value) - elif variable in META_VARS_STR: - det_variables["meta"][variable] = str(value)[:-1].strip() - elif variable in META_UB_MATRIX: - det_variables["meta"][variable] = re.findall( - r"[-+]?\d*\.\d+|\d+", str(value) - ) - except ValueError as error: - print( - "Some values are not in expected format (str or float), error:", str(error) - ) + if variable in META_VARS_FLOAT: + det_variables["meta"][variable] = float(value) + elif variable in META_VARS_STR: + det_variables["meta"][variable] = str(value)[:-1].strip() + elif variable in META_UB_MATRIX: + det_variables["meta"][variable] = re.findall(r"[-+]?\d*\.\d+|\d+", str(value)) elif "#data" in line: if det_variables["file_type"] == "ccl": From 3fd7b5932866d6ac22d7119396515e7bf24a9e83 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 11 Sep 2020 13:52:58 +0200 Subject: [PATCH 015/116] Update load_1D.py --- pyzebra/load_1D.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index e3aaf05..23c984b 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -1,5 +1,6 @@ import re import numpy as np +from decimal import Decimal META_VARS_STR = ( "instrument", @@ -81,6 +82,7 @@ def load_1D(filepath): elif "#data" in line: if det_variables["file_type"] == "ccl": + decimal = list() data = infile.readlines() position = -1 for lines in data: @@ -95,8 +97,11 @@ def load_1D(filepath): measurement_number = int(lines.split()[0]) d = {} d["h_index"] = float(lines.split()[1]) + decimal.append(bool(Decimal(d["h_index"]) % 1 == 0)) d["k_index"] = float(lines.split()[2]) + decimal.append(bool(Decimal(d["k_index"]) % 1 == 0)) d["l_index"] = float(lines.split()[3]) + decimal.append(bool(Decimal(d["l_index"]) % 1 == 0)) if det_variables["meta"]["zebra_mode"] == "bi": d["twotheta_angle"] = float(lines.split()[4]) # gamma d["omega_angle"] = float(lines.split()[5]) # omega @@ -157,5 +162,9 @@ def load_1D(filepath): det_variables["Measurements"]["time"] = time else: print("Unknown file extention") + if all(decimal): + det_variables["meta"]["indices"] = "hkl" + else: + det_variables["meta"]["indices"] = "real" return det_variables From e5609a8cd55c30ee2fb2c899cd6f67132f11c235 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Thu, 10 Sep 2020 15:14:55 +0200 Subject: [PATCH 016/116] Add files via upload function to find peaks (more detailed description in email) --- pyzebra/ccl_findpeaks.py | 81 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 pyzebra/ccl_findpeaks.py diff --git a/pyzebra/ccl_findpeaks.py b/pyzebra/ccl_findpeaks.py new file mode 100644 index 0000000..a0d2fd6 --- /dev/null +++ b/pyzebra/ccl_findpeaks.py @@ -0,0 +1,81 @@ +import numpy as np +import scipy as sc +from scipy.interpolate import interp1d +from scipy.signal import savgol_filter + + +def ccl_findpeaks(data, keys, int_threshold=None, prominence=None, smooth=True, window_size=None, poly_order=None): + + """ function iterates through the dictionary created by load_cclv2 and locates peaks for each measurement + args: data (dictionary from load_cclv2), + + int_threshold - fraction of threshold_intensity/max_intensity, must be positive num between 0 and 1 + i.e. will only detect peaks above 75% of max intensity + + prominence - defines a drop of values that must be between two peaks, must be positive number + i.e. if promimence is 20, it will detect two neigbouring peaks of 300 and 310 intesities, if none of the itermediate values are lower that 290 + + smooth - if true, smooths data by savitzky golay filter, if false - no smoothing + + window_size - window size for savgol filter, must be odd positive integer + + poly_order = order of the polynomial used in savgol filter, must be positive integer smaller than window_size + returns: dictionary with following structure: + D{M34{ 'num_of_peaks': 1, #num of peaks + 'peak_indexes': [20], # index of peaks in omega array + 'peak_heights': [90.], # height of the peaks (if data vere smoothed its the heigh of the peaks in smoothed data) + + + """ + + if type(data) is dict and data["file_type"] == 'ccl': + int_threshold = 0.75 if int_threshold is None else int_threshold + prominence = 50 if prominence is None else prominence + smooth = False if smooth is None else smooth + window_size = 7 if window_size is None else window_size + poly_order = 3 if poly_order is None else poly_order + + if 0 <= int_threshold <= 1: + pass + else: + int_threshold = 0.75 + print('Invalid value for int_threshold, select value between 0 and 1, new value set to:', int_threshold) + if isinstance(window_size, int) is True and (window_size % 2) != 0 and window_size >= 1: + pass + else: + window_size = 7 + print('Invalid value for window_size, select positive odd integer, new value set to:', window_size) + if isinstance(poly_order, int) is True and window_size > poly_order >= 1: + pass + else: + poly_order = 3 + print('Invalid value for poly_order, select positive integer smaller than window_size, new value set to:', poly_order) + if isinstance(prominence, (int, float)) is True and prominence > 0: + pass + else: + prominence = 50 + print('Invalid value for prominence, select positive number, new value set to:', + prominence) + + omega = data["Measurements"][str(keys)]["omega"] + counts = np.array(data["Measurements"][str(keys)]["counts"]) + if smooth is True: + itp = interp1d(omega, counts, kind='linear') + absintensity = [abs(number) for number in counts] + lowest_intensity = min(absintensity) + counts[counts < 0] = lowest_intensity + smooth_peaks = savgol_filter(itp(omega), window_size, poly_order) + + else: + smooth_peaks = counts + + indexes = sc.signal.find_peaks(smooth_peaks, height=int_threshold*max(smooth_peaks), prominence=prominence) + data["Measurements"][str(keys)]["num_of_peaks"] = len(indexes[0]) + data["Measurements"][str(keys)]["peak_indexes"] = indexes[0] + data["Measurements"][str(keys)]["peak_heights"] = indexes[1]["peak_heights"] + data["Measurements"][str(keys)]["smooth_peaks"] = smooth_peaks # smoothed curve + + return data + + else: + return print('Data is not a dictionary or was not made from ccl file') \ No newline at end of file From 8b603c5924a07792cf4725f64adb20b02c13d4d4 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 11 Sep 2020 15:30:30 +0200 Subject: [PATCH 017/116] guard close changed --- pyzebra/ccl_findpeaks.py | 121 +++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 55 deletions(-) diff --git a/pyzebra/ccl_findpeaks.py b/pyzebra/ccl_findpeaks.py index a0d2fd6..a8df1d4 100644 --- a/pyzebra/ccl_findpeaks.py +++ b/pyzebra/ccl_findpeaks.py @@ -4,78 +4,89 @@ from scipy.interpolate import interp1d from scipy.signal import savgol_filter -def ccl_findpeaks(data, keys, int_threshold=None, prominence=None, smooth=True, window_size=None, poly_order=None): +def ccl_findpeaks( + data, keys, int_threshold=None, prominence=None, smooth=True, window_size=None, poly_order=None +): - """ function iterates through the dictionary created by load_cclv2 and locates peaks for each measurement + """function iterates through the dictionary created by load_cclv2 and locates peaks for each measurement args: data (dictionary from load_cclv2), int_threshold - fraction of threshold_intensity/max_intensity, must be positive num between 0 and 1 i.e. will only detect peaks above 75% of max intensity prominence - defines a drop of values that must be between two peaks, must be positive number - i.e. if promimence is 20, it will detect two neigbouring peaks of 300 and 310 intesities, if none of the itermediate values are lower that 290 + i.e. if promimence is 20, it will detect two neigbouring peaks of 300 and 310 intesities, + if none of the itermediate values are lower that 290 smooth - if true, smooths data by savitzky golay filter, if false - no smoothing window_size - window size for savgol filter, must be odd positive integer - poly_order = order of the polynomial used in savgol filter, must be positive integer smaller than window_size - returns: dictionary with following structure: + poly_order = order of the polynomial used in savgol filter, must be positive integer smaller than + window_size returns: dictionary with following structure: D{M34{ 'num_of_peaks': 1, #num of peaks 'peak_indexes': [20], # index of peaks in omega array - 'peak_heights': [90.], # height of the peaks (if data vere smoothed its the heigh of the peaks in smoothed data) + 'peak_heights': [90.], # height of the peaks (if data vere smoothed + its the heigh of the peaks in smoothed data) + """ + if type(data) is not dict and data["file_type"] != "ccl": + print("Data is not a dictionary or was not made from ccl file") - """ + int_threshold = 0.75 if int_threshold is None else int_threshold + prominence = 50 if prominence is None else prominence + smooth = False if smooth is None else smooth + window_size = 7 if window_size is None else window_size + poly_order = 3 if poly_order is None else poly_order - if type(data) is dict and data["file_type"] == 'ccl': - int_threshold = 0.75 if int_threshold is None else int_threshold - prominence = 50 if prominence is None else prominence - smooth = False if smooth is None else smooth - window_size = 7 if window_size is None else window_size - poly_order = 3 if poly_order is None else poly_order + if 0 <= int_threshold <= 1: + pass + else: + int_threshold = 0.75 + print( + "Invalid value for int_threshold, select value between 0 and 1, new value set to:", + int_threshold, + ) + if isinstance(window_size, int) is True and (window_size % 2) != 0 and window_size >= 1: + pass + else: + window_size = 7 + print( + "Invalid value for window_size, select positive odd integer, new value set to:", + window_size, + ) + if isinstance(poly_order, int) is True and window_size > poly_order >= 1: + pass + else: + poly_order = 3 + print( + "Invalid value for poly_order, select positive integer smaller than window_size, new value set to:", + poly_order, + ) + if isinstance(prominence, (int, float)) is True and prominence > 0: + pass + else: + prominence = 50 + print("Invalid value for prominence, select positive number, new value set to:", prominence) - if 0 <= int_threshold <= 1: - pass - else: - int_threshold = 0.75 - print('Invalid value for int_threshold, select value between 0 and 1, new value set to:', int_threshold) - if isinstance(window_size, int) is True and (window_size % 2) != 0 and window_size >= 1: - pass - else: - window_size = 7 - print('Invalid value for window_size, select positive odd integer, new value set to:', window_size) - if isinstance(poly_order, int) is True and window_size > poly_order >= 1: - pass - else: - poly_order = 3 - print('Invalid value for poly_order, select positive integer smaller than window_size, new value set to:', poly_order) - if isinstance(prominence, (int, float)) is True and prominence > 0: - pass - else: - prominence = 50 - print('Invalid value for prominence, select positive number, new value set to:', - prominence) - - omega = data["Measurements"][str(keys)]["omega"] - counts = np.array(data["Measurements"][str(keys)]["counts"]) - if smooth is True: - itp = interp1d(omega, counts, kind='linear') - absintensity = [abs(number) for number in counts] - lowest_intensity = min(absintensity) - counts[counts < 0] = lowest_intensity - smooth_peaks = savgol_filter(itp(omega), window_size, poly_order) - - else: - smooth_peaks = counts - - indexes = sc.signal.find_peaks(smooth_peaks, height=int_threshold*max(smooth_peaks), prominence=prominence) - data["Measurements"][str(keys)]["num_of_peaks"] = len(indexes[0]) - data["Measurements"][str(keys)]["peak_indexes"] = indexes[0] - data["Measurements"][str(keys)]["peak_heights"] = indexes[1]["peak_heights"] - data["Measurements"][str(keys)]["smooth_peaks"] = smooth_peaks # smoothed curve - - return data + omega = data["Measurements"][str(keys)]["omega"] + counts = np.array(data["Measurements"][str(keys)]["counts"]) + if smooth is True: + itp = interp1d(omega, counts, kind="linear") + absintensity = [abs(number) for number in counts] + lowest_intensity = min(absintensity) + counts[counts < 0] = lowest_intensity + smooth_peaks = savgol_filter(itp(omega), window_size, poly_order) else: - return print('Data is not a dictionary or was not made from ccl file') \ No newline at end of file + smooth_peaks = counts + + indexes = sc.signal.find_peaks( + smooth_peaks, height=int_threshold * max(smooth_peaks), prominence=prominence + ) + data["Measurements"][str(keys)]["num_of_peaks"] = len(indexes[0]) + data["Measurements"][str(keys)]["peak_indexes"] = indexes[0] + data["Measurements"][str(keys)]["peak_heights"] = indexes[1]["peak_heights"] + data["Measurements"][str(keys)]["smooth_peaks"] = smooth_peaks # smoothed curve + + return data From 40f897ab58b4d4e61410bfb71530c7a1df557b07 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 11 Sep 2020 16:45:31 +0200 Subject: [PATCH 018/116] Update ccl_findpeaks.py --- pyzebra/ccl_findpeaks.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyzebra/ccl_findpeaks.py b/pyzebra/ccl_findpeaks.py index a8df1d4..6e5ba56 100644 --- a/pyzebra/ccl_findpeaks.py +++ b/pyzebra/ccl_findpeaks.py @@ -5,7 +5,7 @@ from scipy.signal import savgol_filter def ccl_findpeaks( - data, keys, int_threshold=None, prominence=None, smooth=True, window_size=None, poly_order=None + data, keys, int_threshold=0.8, prominence=50, smooth=False, window_size=7, poly_order=3 ): """function iterates through the dictionary created by load_cclv2 and locates peaks for each measurement @@ -33,12 +33,6 @@ def ccl_findpeaks( if type(data) is not dict and data["file_type"] != "ccl": print("Data is not a dictionary or was not made from ccl file") - int_threshold = 0.75 if int_threshold is None else int_threshold - prominence = 50 if prominence is None else prominence - smooth = False if smooth is None else smooth - window_size = 7 if window_size is None else window_size - poly_order = 3 if poly_order is None else poly_order - if 0 <= int_threshold <= 1: pass else: From 661e284b167482d2f92d52415dfd8759f4ffc3e6 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Mon, 14 Sep 2020 09:42:14 +0200 Subject: [PATCH 019/116] Update ccl_findpeaks.py --- pyzebra/ccl_findpeaks.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/pyzebra/ccl_findpeaks.py b/pyzebra/ccl_findpeaks.py index 6e5ba56..fc52425 100644 --- a/pyzebra/ccl_findpeaks.py +++ b/pyzebra/ccl_findpeaks.py @@ -15,54 +15,49 @@ def ccl_findpeaks( i.e. will only detect peaks above 75% of max intensity prominence - defines a drop of values that must be between two peaks, must be positive number - i.e. if promimence is 20, it will detect two neigbouring peaks of 300 and 310 intesities, + i.e. if promimence is 20, it will detect two neigbouring peaks of 300 and 310 intesities, if none of the itermediate values are lower that 290 smooth - if true, smooths data by savitzky golay filter, if false - no smoothing window_size - window size for savgol filter, must be odd positive integer - poly_order = order of the polynomial used in savgol filter, must be positive integer smaller than + poly_order = order of the polynomial used in savgol filter, must be positive integer smaller than window_size returns: dictionary with following structure: D{M34{ 'num_of_peaks': 1, #num of peaks 'peak_indexes': [20], # index of peaks in omega array - 'peak_heights': [90.], # height of the peaks (if data vere smoothed + 'peak_heights': [90.], # height of the peaks (if data vere smoothed its the heigh of the peaks in smoothed data) """ if type(data) is not dict and data["file_type"] != "ccl": print("Data is not a dictionary or was not made from ccl file") - if 0 <= int_threshold <= 1: - pass - else: - int_threshold = 0.75 + if not 0 <= int_threshold <= 1: + int_threshold = 0.8 print( "Invalid value for int_threshold, select value between 0 and 1, new value set to:", int_threshold, ) - if isinstance(window_size, int) is True and (window_size % 2) != 0 and window_size >= 1: - pass - else: + + if isinstance(window_size, int) is False or (window_size % 2) == 0 or window_size <= 1: window_size = 7 print( - "Invalid value for window_size, select positive odd integer, new value set to:", - window_size, - ) - if isinstance(poly_order, int) is True and window_size > poly_order >= 1: - pass - else: + "Invalid value for window_size, select positive odd integer, new value set to!:", + window_size) + + if isinstance(poly_order, int) is False or window_size < poly_order: poly_order = 3 print( "Invalid value for poly_order, select positive integer smaller than window_size, new value set to:", poly_order, ) - if isinstance(prominence, (int, float)) is True and prominence > 0: - pass - else: + + if isinstance(prominence, (int, float)) is False and prominence < 0: prominence = 50 print("Invalid value for prominence, select positive number, new value set to:", prominence) + omega = data["Measurements"][str(keys)]["omega"] counts = np.array(data["Measurements"][str(keys)]["counts"]) if smooth is True: From cc3f1d28a5670dad6fdaec83a19edbee0b1a09ce Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Tue, 15 Sep 2020 15:17:00 +0200 Subject: [PATCH 020/116] Update load_1D.py Corrected the dat beaking bug --- pyzebra/load_1D.py | 140 ++++++++++++++++++++++++--------------------- 1 file changed, 76 insertions(+), 64 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 23c984b..215270c 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -2,57 +2,6 @@ import re import numpy as np from decimal import Decimal -META_VARS_STR = ( - "instrument", - "title", - "sample", - "user", - "ProposalID", - "original_filename", - "date", - "zebra_mode", - "proposal", - "proposal_user", - "proposal_title", - "proposal_email", -) -META_VARS_FLOAT = ( - "mf", - "2-theta", - "chi", - "phi", - "nu", - "temp", - "wavelenght", - "a", - "b", - "c", - "alpha", - "beta", - "gamma", - "cex1", - "cex2", - "mexz", - "moml", - "mcvl", - "momu", - "mcvu", - "detectorDistance", - "snv", - "snh", - "snvm", - "snhm", - "s1vt", - "s1vb", - "s1hr", - "s1hl", - "s2vt", - "s2vb", - "s2hr", - "s2hl", -) -META_UB_MATRIX = ("ub1j", "ub2j", "ub3j") - def load_1D(filepath): """ @@ -63,23 +12,82 @@ def load_1D(filepath): :arg filepath :returns det_variables - dictionary of all detector/scan variables and dictinionary for every measurement. - Names of these dictionaries are M + measurement number. They include HKL indeces, angles, - monitors, stepsize and array of counts + Names of these dictionaries are M + measurement number. They include HKL indeces, angles, monitors, + stepsize and array of counts """ + det_variables = {"file_type": str(filepath)[-3:], "meta": {}} + meta_vars_str = ( + "instrument", + "title", + "sample", + "user", + "ProposalID", + "original_filename", + "date", + "zebra_mode", + "proposal", + "proposal_user", + "proposal_title", + "proposal_email", + ) + meta_vars_float = ( + "mf", + "2-theta", + "chi", + "phi", + "nu", + "temp", + "wavelenght", + "a", + "b", + "c", + "alpha", + "beta", + "gamma", + "cex1", + "cex2", + "mexz", + "moml", + "mcvl", + "momu", + "mcvu", + "detectorDistance", + "snv", + "snh", + "snvm", + "snhm", + "s1vt", + "s1vb", + "s1hr", + "s1hl", + "s2vt", + "s2vb", + "s2hr", + "s2hl", + ) + meta_ub_matrix = ("ub1j", "ub2j", "ub3j") + with open(filepath, "r") as infile: for line in infile: det_variables["Measurements"] = {} if "=" in line: variable, value = line.split("=") variable = variable.strip() - if variable in META_VARS_FLOAT: - det_variables["meta"][variable] = float(value) - elif variable in META_VARS_STR: - det_variables["meta"][variable] = str(value)[:-1].strip() - elif variable in META_UB_MATRIX: - det_variables["meta"][variable] = re.findall(r"[-+]?\d*\.\d+|\d+", str(value)) - + try: + if variable in meta_vars_float: + det_variables["meta"][variable] = float(value) + elif variable in meta_vars_str: + det_variables["meta"][variable] = str(value)[:-1].strip() + elif variable in meta_ub_matrix: + det_variables["meta"][variable] = re.findall( + r"[-+]?\d*\.\d+|\d+", str(value) + ) + except ValueError as error: + print( + "Some values are not in expected format (str or float), error:", + str(error), + ) elif "#data" in line: if det_variables["file_type"] == "ccl": decimal = list() @@ -111,6 +119,7 @@ def load_1D(filepath): d["gamma_angle"] = float(lines.split()[4]) # gamma d["omega_angle"] = float(lines.split()[5]) # omega d["nu_angle"] = float(lines.split()[6]) # nu + d["unkwn_angle"] = float(lines.split()[7]) next_line = data[position + 1] d["number_of_measurements"] = int(next_line.split()[0]) @@ -137,6 +146,12 @@ def load_1D(filepath): ) d["counts"] = counts det_variables["Measurements"][str("M" + str(measurement_number))] = d + if all(decimal): + det_variables["meta"]["indices"] = "hkl" + else: + det_variables["meta"]["indices"] = "real" + print("indeces:", det_variables["meta"]["indices"]) + elif det_variables["file_type"] == "dat": data = infile.readlines() @@ -162,9 +177,6 @@ def load_1D(filepath): det_variables["Measurements"]["time"] = time else: print("Unknown file extention") - if all(decimal): - det_variables["meta"]["indices"] = "hkl" - else: - det_variables["meta"]["indices"] = "real" return det_variables + From ab002be606680d099af688276512e658c467b8ba Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 15 Sep 2020 15:53:24 +0200 Subject: [PATCH 021/116] Revert "Update load_1D.py" This reverts commit cc3f1d28a5670dad6fdaec83a19edbee0b1a09ce. --- pyzebra/load_1D.py | 140 +++++++++++++++++++++------------------------ 1 file changed, 64 insertions(+), 76 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 215270c..23c984b 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -2,6 +2,57 @@ import re import numpy as np from decimal import Decimal +META_VARS_STR = ( + "instrument", + "title", + "sample", + "user", + "ProposalID", + "original_filename", + "date", + "zebra_mode", + "proposal", + "proposal_user", + "proposal_title", + "proposal_email", +) +META_VARS_FLOAT = ( + "mf", + "2-theta", + "chi", + "phi", + "nu", + "temp", + "wavelenght", + "a", + "b", + "c", + "alpha", + "beta", + "gamma", + "cex1", + "cex2", + "mexz", + "moml", + "mcvl", + "momu", + "mcvu", + "detectorDistance", + "snv", + "snh", + "snvm", + "snhm", + "s1vt", + "s1vb", + "s1hr", + "s1hl", + "s2vt", + "s2vb", + "s2hr", + "s2hl", +) +META_UB_MATRIX = ("ub1j", "ub2j", "ub3j") + def load_1D(filepath): """ @@ -12,82 +63,23 @@ def load_1D(filepath): :arg filepath :returns det_variables - dictionary of all detector/scan variables and dictinionary for every measurement. - Names of these dictionaries are M + measurement number. They include HKL indeces, angles, monitors, - stepsize and array of counts + Names of these dictionaries are M + measurement number. They include HKL indeces, angles, + monitors, stepsize and array of counts """ - det_variables = {"file_type": str(filepath)[-3:], "meta": {}} - meta_vars_str = ( - "instrument", - "title", - "sample", - "user", - "ProposalID", - "original_filename", - "date", - "zebra_mode", - "proposal", - "proposal_user", - "proposal_title", - "proposal_email", - ) - meta_vars_float = ( - "mf", - "2-theta", - "chi", - "phi", - "nu", - "temp", - "wavelenght", - "a", - "b", - "c", - "alpha", - "beta", - "gamma", - "cex1", - "cex2", - "mexz", - "moml", - "mcvl", - "momu", - "mcvu", - "detectorDistance", - "snv", - "snh", - "snvm", - "snhm", - "s1vt", - "s1vb", - "s1hr", - "s1hl", - "s2vt", - "s2vb", - "s2hr", - "s2hl", - ) - meta_ub_matrix = ("ub1j", "ub2j", "ub3j") - with open(filepath, "r") as infile: for line in infile: det_variables["Measurements"] = {} if "=" in line: variable, value = line.split("=") variable = variable.strip() - try: - if variable in meta_vars_float: - det_variables["meta"][variable] = float(value) - elif variable in meta_vars_str: - det_variables["meta"][variable] = str(value)[:-1].strip() - elif variable in meta_ub_matrix: - det_variables["meta"][variable] = re.findall( - r"[-+]?\d*\.\d+|\d+", str(value) - ) - except ValueError as error: - print( - "Some values are not in expected format (str or float), error:", - str(error), - ) + if variable in META_VARS_FLOAT: + det_variables["meta"][variable] = float(value) + elif variable in META_VARS_STR: + det_variables["meta"][variable] = str(value)[:-1].strip() + elif variable in META_UB_MATRIX: + det_variables["meta"][variable] = re.findall(r"[-+]?\d*\.\d+|\d+", str(value)) + elif "#data" in line: if det_variables["file_type"] == "ccl": decimal = list() @@ -119,7 +111,6 @@ def load_1D(filepath): d["gamma_angle"] = float(lines.split()[4]) # gamma d["omega_angle"] = float(lines.split()[5]) # omega d["nu_angle"] = float(lines.split()[6]) # nu - d["unkwn_angle"] = float(lines.split()[7]) next_line = data[position + 1] d["number_of_measurements"] = int(next_line.split()[0]) @@ -146,12 +137,6 @@ def load_1D(filepath): ) d["counts"] = counts det_variables["Measurements"][str("M" + str(measurement_number))] = d - if all(decimal): - det_variables["meta"]["indices"] = "hkl" - else: - det_variables["meta"]["indices"] = "real" - print("indeces:", det_variables["meta"]["indices"]) - elif det_variables["file_type"] == "dat": data = infile.readlines() @@ -177,6 +162,9 @@ def load_1D(filepath): det_variables["Measurements"]["time"] = time else: print("Unknown file extention") + if all(decimal): + det_variables["meta"]["indices"] = "hkl" + else: + det_variables["meta"]["indices"] = "real" return det_variables - From 8b3f1b568f76a7c0d04b833fcfa3e36ed6e616db Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 15 Sep 2020 15:59:21 +0200 Subject: [PATCH 022/116] Break metadata and data reads * fix decimal issue * add unkwn_angle --- pyzebra/load_1D.py | 174 +++++++++++++++++++++++---------------------- 1 file changed, 90 insertions(+), 84 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 23c984b..caf96ee 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -68,6 +68,7 @@ def load_1D(filepath): """ det_variables = {"file_type": str(filepath)[-3:], "meta": {}} with open(filepath, "r") as infile: + # read metadata for line in infile: det_variables["Measurements"] = {} if "=" in line: @@ -80,91 +81,96 @@ def load_1D(filepath): elif variable in META_UB_MATRIX: det_variables["meta"][variable] = re.findall(r"[-+]?\d*\.\d+|\d+", str(value)) - elif "#data" in line: - if det_variables["file_type"] == "ccl": - decimal = list() - data = infile.readlines() - position = -1 - for lines in data: - position = position + 1 - if ( - bool(re.match("(\s\s\s\d)", lines[0:4])) == True - or bool(re.match("(\s\s\d\d)", lines[0:4])) == True - or bool(re.match("(\s\d\d\d)", lines[0:4])) == True - or bool(re.match("(\d\d\d\d)", lines[0:4])) == True - ): - counts = [] - measurement_number = int(lines.split()[0]) - d = {} - d["h_index"] = float(lines.split()[1]) - decimal.append(bool(Decimal(d["h_index"]) % 1 == 0)) - d["k_index"] = float(lines.split()[2]) - decimal.append(bool(Decimal(d["k_index"]) % 1 == 0)) - d["l_index"] = float(lines.split()[3]) - decimal.append(bool(Decimal(d["l_index"]) % 1 == 0)) - if det_variables["meta"]["zebra_mode"] == "bi": - d["twotheta_angle"] = float(lines.split()[4]) # gamma - d["omega_angle"] = float(lines.split()[5]) # omega - d["chi_angle"] = float(lines.split()[6]) # nu - d["phi_angle"] = float(lines.split()[7]) # doesnt matter - elif det_variables["meta"]["zebra_mode"] == "nb": - d["gamma_angle"] = float(lines.split()[4]) # gamma - d["omega_angle"] = float(lines.split()[5]) # omega - d["nu_angle"] = float(lines.split()[6]) # nu + if "#data" in line: + # this is the end of metadata and the start of data section + break - next_line = data[position + 1] - d["number_of_measurements"] = int(next_line.split()[0]) - d["angle_step"] = float(next_line.split()[1]) - d["monitor"] = float(next_line.split()[2]) - d["unkwn1"] = float(next_line.split()[3]) - d["unkwn2"] = float(next_line.split()[4]) - d["date"] = str(next_line.split()[5]) - d["time"] = str(next_line.split()[6]) - d["scan_type"] = str(next_line.split()[7]) - for i in range( - int(int(next_line.split()[0]) / 10) - + (int(next_line.split()[0]) % 10 > 0) - ): - fileline = data[position + 2 + i].split() - numbers = [int(w) for w in fileline] - counts = counts + numbers - d["omega"] = np.linspace( - float(lines.split()[5]) - - (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), - float(lines.split()[5]) - + (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), - int(next_line.split()[0]), - ) - d["counts"] = counts - det_variables["Measurements"][str("M" + str(measurement_number))] = d - - elif det_variables["file_type"] == "dat": - data = infile.readlines() - num_of_points = int(data[1].split()[0]) - omega = [] + # read data + if det_variables["file_type"] == "ccl": + decimal = list() + data = infile.readlines() + position = -1 + for lines in data: + position = position + 1 + if ( + bool(re.match("(\s\s\s\d)", lines[0:4])) == True + or bool(re.match("(\s\s\d\d)", lines[0:4])) == True + or bool(re.match("(\s\d\d\d)", lines[0:4])) == True + or bool(re.match("(\d\d\d\d)", lines[0:4])) == True + ): counts = [] - monitor1 = [] - monitor2 = [] - monitor3 = [] - time = [] - for position in range(num_of_points): - omega.append(float(data[position + 3].split()[1])) - counts.append(float(data[position + 3].split()[2])) - monitor1.append(float(data[position + 3].split()[3])) - monitor2.append(float(data[position + 3].split()[4])) - monitor3.append(float(data[position + 3].split()[5])) - time.append(float(data[position + 3].split()[6])) - det_variables["Measurements"]["omega"] = omega - det_variables["Measurements"]["counts"] = counts - det_variables["Measurements"]["Monitor1"] = monitor1 - det_variables["Measurements"]["Monitor2"] = monitor2 - det_variables["Measurements"]["Monitor3"] = monitor3 - det_variables["Measurements"]["time"] = time - else: - print("Unknown file extention") - if all(decimal): - det_variables["meta"]["indices"] = "hkl" - else: - det_variables["meta"]["indices"] = "real" + measurement_number = int(lines.split()[0]) + d = {} + d["h_index"] = float(lines.split()[1]) + decimal.append(bool(Decimal(d["h_index"]) % 1 == 0)) + d["k_index"] = float(lines.split()[2]) + decimal.append(bool(Decimal(d["k_index"]) % 1 == 0)) + d["l_index"] = float(lines.split()[3]) + decimal.append(bool(Decimal(d["l_index"]) % 1 == 0)) + if det_variables["meta"]["zebra_mode"] == "bi": + d["twotheta_angle"] = float(lines.split()[4]) # gamma + d["omega_angle"] = float(lines.split()[5]) # omega + d["chi_angle"] = float(lines.split()[6]) # nu + d["phi_angle"] = float(lines.split()[7]) # doesnt matter + elif det_variables["meta"]["zebra_mode"] == "nb": + d["gamma_angle"] = float(lines.split()[4]) # gamma + d["omega_angle"] = float(lines.split()[5]) # omega + d["nu_angle"] = float(lines.split()[6]) # nu + d["unkwn_angle"] = float(lines.split()[7]) + + next_line = data[position + 1] + d["number_of_measurements"] = int(next_line.split()[0]) + d["angle_step"] = float(next_line.split()[1]) + d["monitor"] = float(next_line.split()[2]) + d["unkwn1"] = float(next_line.split()[3]) + d["unkwn2"] = float(next_line.split()[4]) + d["date"] = str(next_line.split()[5]) + d["time"] = str(next_line.split()[6]) + d["scan_type"] = str(next_line.split()[7]) + for i in range( + int(int(next_line.split()[0]) / 10) + (int(next_line.split()[0]) % 10 > 0) + ): + fileline = data[position + 2 + i].split() + numbers = [int(w) for w in fileline] + counts = counts + numbers + d["omega"] = np.linspace( + float(lines.split()[5]) + - (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), + float(lines.split()[5]) + + (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), + int(next_line.split()[0]), + ) + d["counts"] = counts + det_variables["Measurements"][str("M" + str(measurement_number))] = d + + if all(decimal): + det_variables["meta"]["indices"] = "hkl" + else: + det_variables["meta"]["indices"] = "real" + + elif det_variables["file_type"] == "dat": + data = infile.readlines() + num_of_points = int(data[1].split()[0]) + omega = [] + counts = [] + monitor1 = [] + monitor2 = [] + monitor3 = [] + time = [] + for position in range(num_of_points): + omega.append(float(data[position + 3].split()[1])) + counts.append(float(data[position + 3].split()[2])) + monitor1.append(float(data[position + 3].split()[3])) + monitor2.append(float(data[position + 3].split()[4])) + monitor3.append(float(data[position + 3].split()[5])) + time.append(float(data[position + 3].split()[6])) + det_variables["Measurements"]["omega"] = omega + det_variables["Measurements"]["counts"] = counts + det_variables["Measurements"]["Monitor1"] = monitor1 + det_variables["Measurements"]["Monitor2"] = monitor2 + det_variables["Measurements"]["Monitor3"] = monitor3 + det_variables["Measurements"]["time"] = time + else: + print("Unknown file extention") return det_variables From fe1d8044a3f2f6e24927a4f8baf4aa0894a56140 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 15 Sep 2020 16:07:41 +0200 Subject: [PATCH 023/116] Simplify dat file processing * rename "omega" -> "om", and "counts" -> "Counts" for consistency with the column names --- pyzebra/ccl_findpeaks.py | 4 ++-- pyzebra/fit2.py | 4 ++-- pyzebra/load_1D.py | 44 ++++++++++++++++++---------------------- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/pyzebra/ccl_findpeaks.py b/pyzebra/ccl_findpeaks.py index fc52425..e141f95 100644 --- a/pyzebra/ccl_findpeaks.py +++ b/pyzebra/ccl_findpeaks.py @@ -58,8 +58,8 @@ def ccl_findpeaks( print("Invalid value for prominence, select positive number, new value set to:", prominence) - omega = data["Measurements"][str(keys)]["omega"] - counts = np.array(data["Measurements"][str(keys)]["counts"]) + omega = data["Measurements"][str(keys)]["om"] + counts = np.array(data["Measurements"][str(keys)]["Counts"]) if smooth is True: itp = interp1d(omega, counts, kind="linear") absintensity = [abs(number) for number in counts] diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index ac499e3..be5fc22 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -40,8 +40,8 @@ def fitccl( print("NO PEAK or more than 1 peak") return - x = list(data["Measurements"][str(keys)]["omega"]) - y = list(data["Measurements"][str(keys)]["counts"]) + x = list(data["Measurements"][str(keys)]["om"]) + y = list(data["Measurements"][str(keys)]["Counts"]) peak_index = data["Measurements"][str(keys)]["peak_indexes"] peak_height = data["Measurements"][str(keys)]["peak_heights"] print("before", constraints_min) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index caf96ee..30ff5b8 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -1,7 +1,9 @@ import re -import numpy as np +from collections import defaultdict from decimal import Decimal +import numpy as np + META_VARS_STR = ( "instrument", "title", @@ -133,14 +135,14 @@ def load_1D(filepath): fileline = data[position + 2 + i].split() numbers = [int(w) for w in fileline] counts = counts + numbers - d["omega"] = np.linspace( + d["om"] = np.linspace( float(lines.split()[5]) - (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), float(lines.split()[5]) + (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), int(next_line.split()[0]), ) - d["counts"] = counts + d["Counts"] = counts det_variables["Measurements"][str("M" + str(measurement_number))] = d if all(decimal): @@ -149,27 +151,21 @@ def load_1D(filepath): det_variables["meta"]["indices"] = "real" elif det_variables["file_type"] == "dat": - data = infile.readlines() - num_of_points = int(data[1].split()[0]) - omega = [] - counts = [] - monitor1 = [] - monitor2 = [] - monitor3 = [] - time = [] - for position in range(num_of_points): - omega.append(float(data[position + 3].split()[1])) - counts.append(float(data[position + 3].split()[2])) - monitor1.append(float(data[position + 3].split()[3])) - monitor2.append(float(data[position + 3].split()[4])) - monitor3.append(float(data[position + 3].split()[5])) - time.append(float(data[position + 3].split()[6])) - det_variables["Measurements"]["omega"] = omega - det_variables["Measurements"]["counts"] = counts - det_variables["Measurements"]["Monitor1"] = monitor1 - det_variables["Measurements"]["Monitor2"] = monitor2 - det_variables["Measurements"]["Monitor3"] = monitor3 - det_variables["Measurements"]["time"] = time + # skip the first 2 rows, the third row contans the column names + next(infile) + next(infile) + row_names = next(infile).split() + + data_cols = defaultdict(list) + for line in infile: + if "END-OF-DATA" in line: + # this is the end of data + break + + for name, val in zip(row_names, line.split()): + data_cols[name].append(float(val)) + + det_variables["Measurements"] = dict(data_cols) else: print("Unknown file extention") From a48b1f64c1fa3cd0dd2ce35576380d2f711a35ae Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 15 Sep 2020 18:03:28 +0200 Subject: [PATCH 024/116] Use a ref to the current measurement --- pyzebra/ccl_findpeaks.py | 17 +++++++------- pyzebra/comm_export.py | 49 +++++++++++++++++++--------------------- pyzebra/fit2.py | 13 ++++++----- 3 files changed, 39 insertions(+), 40 deletions(-) diff --git a/pyzebra/ccl_findpeaks.py b/pyzebra/ccl_findpeaks.py index e141f95..adcd614 100644 --- a/pyzebra/ccl_findpeaks.py +++ b/pyzebra/ccl_findpeaks.py @@ -29,6 +29,7 @@ def ccl_findpeaks( 'peak_heights': [90.], # height of the peaks (if data vere smoothed its the heigh of the peaks in smoothed data) """ + meas = data["Measurements"][keys] if type(data) is not dict and data["file_type"] != "ccl": print("Data is not a dictionary or was not made from ccl file") @@ -44,7 +45,8 @@ def ccl_findpeaks( window_size = 7 print( "Invalid value for window_size, select positive odd integer, new value set to!:", - window_size) + window_size, + ) if isinstance(poly_order, int) is False or window_size < poly_order: poly_order = 3 @@ -57,9 +59,8 @@ def ccl_findpeaks( prominence = 50 print("Invalid value for prominence, select positive number, new value set to:", prominence) - - omega = data["Measurements"][str(keys)]["om"] - counts = np.array(data["Measurements"][str(keys)]["Counts"]) + omega = meas["om"] + counts = np.array(meas["Counts"]) if smooth is True: itp = interp1d(omega, counts, kind="linear") absintensity = [abs(number) for number in counts] @@ -73,9 +74,9 @@ def ccl_findpeaks( indexes = sc.signal.find_peaks( smooth_peaks, height=int_threshold * max(smooth_peaks), prominence=prominence ) - data["Measurements"][str(keys)]["num_of_peaks"] = len(indexes[0]) - data["Measurements"][str(keys)]["peak_indexes"] = indexes[0] - data["Measurements"][str(keys)]["peak_heights"] = indexes[1]["peak_heights"] - data["Measurements"][str(keys)]["smooth_peaks"] = smooth_peaks # smoothed curve + meas["num_of_peaks"] = len(indexes[0]) + meas["peak_indexes"] = indexes[0] + meas["peak_heights"] = indexes[1]["peak_heights"] + meas["smooth_peaks"] = smooth_peaks # smoothed curve return data diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py index eac4c8b..3209821 100644 --- a/pyzebra/comm_export.py +++ b/pyzebra/comm_export.py @@ -30,36 +30,34 @@ def export_comm(data, path, lorentz=False): padding = [4, 6, 10, 8] with open(str(path + extension), "w") as out_file: - for keys in data["Measurements"]: - print(keys) + for meas_num, meas in data["Measurements"].items(): + print(meas_num) try: - meas_number_str = f"{keys[1:]:{align}{padding[0]}}" - h_str = f'{int(data["Measurements"][str(keys)]["h_index"]):{padding[1]}}' - k_str = f'{int(data["Measurements"][str(keys)]["k_index"]):{padding[1]}}' - l_str = f'{int(data["Measurements"][str(keys)]["l_index"]):{padding[1]}}' - if data["Measurements"][str(keys)]["fit"]["export_fit"] is True: - area = float(data["Measurements"][str(keys)]["fit"]["g_amp"][0]) + float( - data["Measurements"][str(keys)]["fit"]["l_amp"][0] - ) + meas_number_str = f"{meas_num[1:]:{align}{padding[0]}}" + h_str = f'{int(meas["h_index"]):{padding[1]}}' + k_str = f'{int(meas["k_index"]):{padding[1]}}' + l_str = f'{int(meas["l_index"]):{padding[1]}}' + if meas["fit"]["export_fit"] is True: + area = float(meas["fit"]["g_amp"][0]) + float(meas["fit"]["l_amp"][0]) else: - area = float(data["Measurements"][str(keys)]["fit"]["int_area"]) - float( - data["Measurements"][str(keys)]["fit"]["int_background"][0] - ) + area = float(meas["fit"]["int_area"]) - float(meas["fit"]["int_background"][0]) if data["meta"]["zebra_mode"] == "bi": - int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], data["Measurements"][str(keys)]["twotheta_angle"])):{align}{padding[2]}}' - angle_str1 = f'{data["Measurements"][str(keys)]["twotheta_angle"]:{padding[3]}}' - angle_str2 = f'{data["Measurements"][str(keys)]["omega_angle"]:{padding[3]}}' - angle_str3 = f'{data["Measurements"][str(keys)]["chi_angle"]:{padding[3]}}' - angle_str4 = f'{data["Measurements"][str(keys)]["phi_angle"]:{padding[3]}}' + int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], meas["twotheta_angle"])):{align}{padding[2]}}' + angle_str1 = f'{meas["twotheta_angle"]:{padding[3]}}' + angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' + angle_str3 = f'{meas["chi_angle"]:{padding[3]}}' + angle_str4 = f'{meas["phi_angle"]:{padding[3]}}' elif data["meta"]["zebra_mode"] == "nb": - int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], data["Measurements"][str(keys)]["gamma_angle"],data["Measurements"][str(keys)]["nu_angle"])):{align}{padding[2]}}' - angle_str1 = f'{data["Measurements"][str(keys)]["gamma_angle"]:{padding[3]}}' - angle_str2 = f'{data["Measurements"][str(keys)]["omega_angle"]:{padding[3]}}' - angle_str3 = f'{data["Measurements"][str(keys)]["nu_angle"]:{padding[3]}}' - angle_str4 = f'{data["Measurements"][str(keys)]["unkwn_angle"]:{padding[3]}}' + int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], meas["gamma_angle"],meas["nu_angle"])):{align}{padding[2]}}' + angle_str1 = f'{meas["gamma_angle"]:{padding[3]}}' + angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' + angle_str3 = f'{meas["nu_angle"]:{padding[3]}}' + angle_str4 = f'{meas["unkwn_angle"]:{padding[3]}}' - sigma_str = f'{"{:8.2f}".format(float(data["Measurements"][str(keys)]["fit"]["g_width"][0])):{align}{padding[2]}}' + sigma_str = ( + f'{"{:8.2f}".format(float(meas["fit"]["g_width"][0])):{align}{padding[2]}}' + ) line = ( meas_number_str + h_str @@ -76,5 +74,4 @@ def export_comm(data, path, lorentz=False): out_file.write(line) except KeyError: - print("Measurement skipped - no fit value for:", keys) - + print("Measurement skipped - no fit value for:", meas_num) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index be5fc22..c86d74b 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -35,15 +35,16 @@ def fitccl( constraints_min = [23, None, 50, None, None, None, 0, 0] constraints_min = [80, None, 1000, None, None, None, 0, 100] """ + meas = data["Measurements"][keys] - if len(data["Measurements"][str(keys)]["peak_indexes"]) != 1: + if len(meas["peak_indexes"]) != 1: print("NO PEAK or more than 1 peak") return - x = list(data["Measurements"][str(keys)]["om"]) - y = list(data["Measurements"][str(keys)]["Counts"]) - peak_index = data["Measurements"][str(keys)]["peak_indexes"] - peak_height = data["Measurements"][str(keys)]["peak_heights"] + x = list(meas["om"]) + y = list(meas["Counts"]) + peak_index = meas["peak_indexes"] + peak_height = meas["peak_heights"] print("before", constraints_min) guess[0] = x[int(peak_index)] if guess[0] is None else guess[0] guess[1] = 0.1 if guess[1] is None else guess[1] @@ -151,6 +152,6 @@ def fitccl( d["int_area"] = num_int_area d["int_background"] = num_int_bacground d["full_report"] = result.fit_report() - data["Measurements"][str(keys)]["fit"] = d + meas["fit"] = d return data From 2f4e097a68394b5a2a53ddd0f423a56185d9a849 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 15 Sep 2020 18:11:21 +0200 Subject: [PATCH 025/116] Check fit results instead of excepting KeyError --- pyzebra/comm_export.py | 82 +++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py index 3209821..27dc80a 100644 --- a/pyzebra/comm_export.py +++ b/pyzebra/comm_export.py @@ -32,46 +32,44 @@ def export_comm(data, path, lorentz=False): with open(str(path + extension), "w") as out_file: for meas_num, meas in data["Measurements"].items(): print(meas_num) - try: - meas_number_str = f"{meas_num[1:]:{align}{padding[0]}}" - h_str = f'{int(meas["h_index"]):{padding[1]}}' - k_str = f'{int(meas["k_index"]):{padding[1]}}' - l_str = f'{int(meas["l_index"]):{padding[1]}}' - if meas["fit"]["export_fit"] is True: - area = float(meas["fit"]["g_amp"][0]) + float(meas["fit"]["l_amp"][0]) - else: - area = float(meas["fit"]["int_area"]) - float(meas["fit"]["int_background"][0]) - - if data["meta"]["zebra_mode"] == "bi": - int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], meas["twotheta_angle"])):{align}{padding[2]}}' - angle_str1 = f'{meas["twotheta_angle"]:{padding[3]}}' - angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' - angle_str3 = f'{meas["chi_angle"]:{padding[3]}}' - angle_str4 = f'{meas["phi_angle"]:{padding[3]}}' - elif data["meta"]["zebra_mode"] == "nb": - int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], meas["gamma_angle"],meas["nu_angle"])):{align}{padding[2]}}' - angle_str1 = f'{meas["gamma_angle"]:{padding[3]}}' - angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' - angle_str3 = f'{meas["nu_angle"]:{padding[3]}}' - angle_str4 = f'{meas["unkwn_angle"]:{padding[3]}}' - - sigma_str = ( - f'{"{:8.2f}".format(float(meas["fit"]["g_width"][0])):{align}{padding[2]}}' - ) - line = ( - meas_number_str - + h_str - + l_str - + k_str - + int_str - + sigma_str - + angle_str1 - + angle_str2 - + angle_str3 - + angle_str4 - + "\n" - ) - out_file.write(line) - - except KeyError: + if meas.get("fit") is None: print("Measurement skipped - no fit value for:", meas_num) + continue + + meas_number_str = f"{meas_num[1:]:{align}{padding[0]}}" + h_str = f'{int(meas["h_index"]):{padding[1]}}' + k_str = f'{int(meas["k_index"]):{padding[1]}}' + l_str = f'{int(meas["l_index"]):{padding[1]}}' + if meas["fit"]["export_fit"] is True: + area = float(meas["fit"]["g_amp"][0]) + float(meas["fit"]["l_amp"][0]) + else: + area = float(meas["fit"]["int_area"]) - float(meas["fit"]["int_background"][0]) + + if data["meta"]["zebra_mode"] == "bi": + int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], meas["twotheta_angle"])):{align}{padding[2]}}' + angle_str1 = f'{meas["twotheta_angle"]:{padding[3]}}' + angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' + angle_str3 = f'{meas["chi_angle"]:{padding[3]}}' + angle_str4 = f'{meas["phi_angle"]:{padding[3]}}' + elif data["meta"]["zebra_mode"] == "nb": + int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], meas["gamma_angle"],meas["nu_angle"])):{align}{padding[2]}}' + angle_str1 = f'{meas["gamma_angle"]:{padding[3]}}' + angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' + angle_str3 = f'{meas["nu_angle"]:{padding[3]}}' + angle_str4 = f'{meas["unkwn_angle"]:{padding[3]}}' + + sigma_str = f'{"{:8.2f}".format(float(meas["fit"]["g_width"][0])):{align}{padding[2]}}' + line = ( + meas_number_str + + h_str + + l_str + + k_str + + int_str + + sigma_str + + angle_str1 + + angle_str2 + + angle_str3 + + angle_str4 + + "\n" + ) + out_file.write(line) From 321e3e83a420eb49f367c91c6ff259ca5f1e9ad6 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Wed, 16 Sep 2020 16:22:17 +0200 Subject: [PATCH 026/116] Add parse_1D function --- pyzebra/load_1D.py | 207 +++++++++++++++++++++++---------------------- 1 file changed, 108 insertions(+), 99 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 30ff5b8..cd1c75b 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -1,3 +1,4 @@ +import os import re from collections import defaultdict from decimal import Decimal @@ -68,105 +69,113 @@ def load_1D(filepath): Names of these dictionaries are M + measurement number. They include HKL indeces, angles, monitors, stepsize and array of counts """ - det_variables = {"file_type": str(filepath)[-3:], "meta": {}} with open(filepath, "r") as infile: - # read metadata - for line in infile: - det_variables["Measurements"] = {} - if "=" in line: - variable, value = line.split("=") - variable = variable.strip() - if variable in META_VARS_FLOAT: - det_variables["meta"][variable] = float(value) - elif variable in META_VARS_STR: - det_variables["meta"][variable] = str(value)[:-1].strip() - elif variable in META_UB_MATRIX: - det_variables["meta"][variable] = re.findall(r"[-+]?\d*\.\d+|\d+", str(value)) - - if "#data" in line: - # this is the end of metadata and the start of data section - break - - # read data - if det_variables["file_type"] == "ccl": - decimal = list() - data = infile.readlines() - position = -1 - for lines in data: - position = position + 1 - if ( - bool(re.match("(\s\s\s\d)", lines[0:4])) == True - or bool(re.match("(\s\s\d\d)", lines[0:4])) == True - or bool(re.match("(\s\d\d\d)", lines[0:4])) == True - or bool(re.match("(\d\d\d\d)", lines[0:4])) == True - ): - counts = [] - measurement_number = int(lines.split()[0]) - d = {} - d["h_index"] = float(lines.split()[1]) - decimal.append(bool(Decimal(d["h_index"]) % 1 == 0)) - d["k_index"] = float(lines.split()[2]) - decimal.append(bool(Decimal(d["k_index"]) % 1 == 0)) - d["l_index"] = float(lines.split()[3]) - decimal.append(bool(Decimal(d["l_index"]) % 1 == 0)) - if det_variables["meta"]["zebra_mode"] == "bi": - d["twotheta_angle"] = float(lines.split()[4]) # gamma - d["omega_angle"] = float(lines.split()[5]) # omega - d["chi_angle"] = float(lines.split()[6]) # nu - d["phi_angle"] = float(lines.split()[7]) # doesnt matter - elif det_variables["meta"]["zebra_mode"] == "nb": - d["gamma_angle"] = float(lines.split()[4]) # gamma - d["omega_angle"] = float(lines.split()[5]) # omega - d["nu_angle"] = float(lines.split()[6]) # nu - d["unkwn_angle"] = float(lines.split()[7]) - - next_line = data[position + 1] - d["number_of_measurements"] = int(next_line.split()[0]) - d["angle_step"] = float(next_line.split()[1]) - d["monitor"] = float(next_line.split()[2]) - d["unkwn1"] = float(next_line.split()[3]) - d["unkwn2"] = float(next_line.split()[4]) - d["date"] = str(next_line.split()[5]) - d["time"] = str(next_line.split()[6]) - d["scan_type"] = str(next_line.split()[7]) - for i in range( - int(int(next_line.split()[0]) / 10) + (int(next_line.split()[0]) % 10 > 0) - ): - fileline = data[position + 2 + i].split() - numbers = [int(w) for w in fileline] - counts = counts + numbers - d["om"] = np.linspace( - float(lines.split()[5]) - - (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), - float(lines.split()[5]) - + (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), - int(next_line.split()[0]), - ) - d["Counts"] = counts - det_variables["Measurements"][str("M" + str(measurement_number))] = d - - if all(decimal): - det_variables["meta"]["indices"] = "hkl" - else: - det_variables["meta"]["indices"] = "real" - - elif det_variables["file_type"] == "dat": - # skip the first 2 rows, the third row contans the column names - next(infile) - next(infile) - row_names = next(infile).split() - - data_cols = defaultdict(list) - for line in infile: - if "END-OF-DATA" in line: - # this is the end of data - break - - for name, val in zip(row_names, line.split()): - data_cols[name].append(float(val)) - - det_variables["Measurements"] = dict(data_cols) - else: - print("Unknown file extention") + _, ext = os.path.splitext(filepath) + det_variables = parse_1D(infile, data_type=ext) return det_variables + + +def parse_1D(fileobj, data_type): + # read metadata + metadata = {} + for line in fileobj: + if "=" in line: + variable, value = line.split("=") + variable = variable.strip() + if variable in META_VARS_FLOAT: + metadata[variable] = float(value) + elif variable in META_VARS_STR: + metadata[variable] = str(value)[:-1].strip() + elif variable in META_UB_MATRIX: + metadata[variable] = re.findall(r"[-+]?\d*\.\d+|\d+", str(value)) + + if "#data" in line: + # this is the end of metadata and the start of data section + break + + # read data + if data_type == ".ccl": + measurements = {} + decimal = list() + data = fileobj.readlines() + position = -1 + for lines in data: + position = position + 1 + if ( + bool(re.match("(\s\s\s\d)", lines[0:4])) == True + or bool(re.match("(\s\s\d\d)", lines[0:4])) == True + or bool(re.match("(\s\d\d\d)", lines[0:4])) == True + or bool(re.match("(\d\d\d\d)", lines[0:4])) == True + ): + counts = [] + measurement_number = int(lines.split()[0]) + d = {} + d["h_index"] = float(lines.split()[1]) + decimal.append(bool(Decimal(d["h_index"]) % 1 == 0)) + d["k_index"] = float(lines.split()[2]) + decimal.append(bool(Decimal(d["k_index"]) % 1 == 0)) + d["l_index"] = float(lines.split()[3]) + decimal.append(bool(Decimal(d["l_index"]) % 1 == 0)) + if metadata["zebra_mode"] == "bi": + d["twotheta_angle"] = float(lines.split()[4]) # gamma + d["omega_angle"] = float(lines.split()[5]) # omega + d["chi_angle"] = float(lines.split()[6]) # nu + d["phi_angle"] = float(lines.split()[7]) # doesnt matter + elif metadata["zebra_mode"] == "nb": + d["gamma_angle"] = float(lines.split()[4]) # gamma + d["omega_angle"] = float(lines.split()[5]) # omega + d["nu_angle"] = float(lines.split()[6]) # nu + d["unkwn_angle"] = float(lines.split()[7]) + + next_line = data[position + 1] + d["number_of_measurements"] = int(next_line.split()[0]) + d["angle_step"] = float(next_line.split()[1]) + d["monitor"] = float(next_line.split()[2]) + d["unkwn1"] = float(next_line.split()[3]) + d["unkwn2"] = float(next_line.split()[4]) + d["date"] = str(next_line.split()[5]) + d["time"] = str(next_line.split()[6]) + d["scan_type"] = str(next_line.split()[7]) + for i in range( + int(int(next_line.split()[0]) / 10) + (int(next_line.split()[0]) % 10 > 0) + ): + fileline = data[position + 2 + i].split() + numbers = [int(w) for w in fileline] + counts = counts + numbers + d["om"] = np.linspace( + float(lines.split()[5]) + - (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), + float(lines.split()[5]) + + (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), + int(next_line.split()[0]), + ) + d["Counts"] = counts + measurements[str("M" + str(measurement_number))] = d + + if all(decimal): + metadata["indices"] = "hkl" + else: + metadata["indices"] = "real" + + elif data_type == ".dat": + # skip the first 2 rows, the third row contans the column names + next(fileobj) + next(fileobj) + col_names = next(fileobj).split() + + data_cols = defaultdict(list) + for line in fileobj: + if "END-OF-DATA" in line: + # this is the end of data + break + + for name, val in zip(col_names, line.split()): + data_cols[name].append(float(val)) + + measurements = dict(data_cols) + + else: + print("Unknown file extention") + + return {"meta": metadata, "Measurements": measurements} From 4d571d2fd65830165a0e86d3cb3154b7cdfba6bb Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Wed, 16 Sep 2020 16:44:55 +0200 Subject: [PATCH 027/116] Import new functions on pyzebra import --- pyzebra/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyzebra/__init__.py b/pyzebra/__init__.py index abd6e90..469fd9f 100644 --- a/pyzebra/__init__.py +++ b/pyzebra/__init__.py @@ -1,3 +1,7 @@ from pyzebra.anatric import * from pyzebra.h5 import * from pyzebra.xtal import * +from pyzebra.load_1D import load_1D, parse_1D +from pyzebra.ccl_findpeaks import ccl_findpeaks +from pyzebra.fit2 import fitccl +from pyzebra.comm_export import export_comm From b0f692a02c305f2c16c45949e4f66a8aabb12db1 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Wed, 16 Sep 2020 17:01:32 +0200 Subject: [PATCH 028/116] Add panel placeholder for 1D detector processing --- pyzebra/app/app.py | 4 ++- pyzebra/app/panel_1D_detector.py | 58 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 pyzebra/app/panel_1D_detector.py diff --git a/pyzebra/app/app.py b/pyzebra/app/app.py index 078c729..7beb17a 100644 --- a/pyzebra/app/app.py +++ b/pyzebra/app/app.py @@ -5,6 +5,7 @@ from bokeh.models import Tabs import panel_anatric import panel_data_viewer +import panel_1D_detector parser = argparse.ArgumentParser( prog="pyzebra", formatter_class=argparse.ArgumentDefaultsHelpFormatter @@ -18,5 +19,6 @@ doc.title = "pyzebra" # Final layout tab_data_viewer = panel_data_viewer.create() tab_anatric = panel_anatric.create() +tab_1D_detector = panel_1D_detector.create() -doc.add_root(Tabs(tabs=[tab_data_viewer, tab_anatric])) +doc.add_root(Tabs(tabs=[tab_data_viewer, tab_anatric, tab_1D_detector])) diff --git a/pyzebra/app/panel_1D_detector.py b/pyzebra/app/panel_1D_detector.py new file mode 100644 index 0000000..52d72f0 --- /dev/null +++ b/pyzebra/app/panel_1D_detector.py @@ -0,0 +1,58 @@ +import base64 +import io +import os + +from bokeh.layouts import column, row +from bokeh.models import ( + BasicTicker, + ColumnDataSource, + DataRange1d, + Div, + FileInput, + Grid, + Line, + LinearAxis, + Panel, + Plot, + Spacer, +) + +import pyzebra + + +def create(): + def upload_button_callback(_attr, _old, new): + with io.StringIO(base64.b64decode(new).decode()) as file: + _, ext = os.path.splitext(upload_button.filename) + res = pyzebra.parse_1D(file, ext) + for _, meas in res["Measurements"].items(): + y = meas["Counts"] + x = list(range(len(y))) + break + + plot_line_source.data.update(x=x, y=y) + + upload_button = FileInput(accept=".ccl") + upload_button.on_change("value", upload_button_callback) + + plot = Plot( + x_range=DataRange1d(), + y_range=DataRange1d(), + plot_height=400, + plot_width=600, + toolbar_location=None, + ) + + plot.add_layout(LinearAxis(axis_label="Counts"), place="left") + plot.add_layout(LinearAxis(), place="below") + + plot.add_layout(Grid(dimension=0, ticker=BasicTicker())) + plot.add_layout(Grid(dimension=1, ticker=BasicTicker())) + + plot_line_source = ColumnDataSource(dict(x=[0], y=[0])) + plot.add_glyph(plot_line_source, Line(x="x", y="y", line_color="steelblue")) + + upload_div = Div(text="Upload .ccl file:") + tab_layout = column(row(column(Spacer(height=5), upload_div), upload_button), plot) + + return Panel(child=tab_layout, title="1D Detector") From bcb4ef9849a68ee4ed38be845b374f6ff71803e8 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Mon, 21 Sep 2020 11:12:49 +0200 Subject: [PATCH 029/116] Update fit2.py Added uncertainty propagation to the numerical integration and weighted fit. Case for 'no peak' scenario also added. Parameter export_fit in meas["fit"] added which is decides if fitted area or numerically integrated area will be saved in comm/incomm files. --- pyzebra/fit2.py | 260 ++++++++++++++++++++++++++++++------------------ 1 file changed, 164 insertions(+), 96 deletions(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index c86d74b..49047a2 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -1,21 +1,37 @@ import numpy as np from lmfit import Model, Parameters -from scipy import integrate from scipy.integrate import simps +import uncertainties as u + def find_nearest(array, value): + # find nearest value and return index array = np.asarray(array) idx = (np.abs(array - value)).argmin() return idx +def create_uncertanities(y, y_err): + # create array with uncertanities for error propagation + combined = np.array([]) + for i in range(len(y)): + part = u.ufloat(y[i], y_err[i]) + combined = np.append(combined, part) + return combined + + def fitccl( - data, keys, guess, vary, constraints_min, constraints_max, numfit_min=None, numfit_max=None + data, + keys, + guess, + vary, + constraints_min, + constraints_max, + numfit_min=None, + numfit_max=None, ): - """Made for fitting of ccl date where 1 peak is expected. Allows for combination of gaussian, lorentzian and linear model combination - - + """Made for fitting of ccl date where 1 peak is expected. Allows for combination of gaussian and linear model combination :param data: dictionary after peak fining :param keys: name of the measurement in the data dict (i.e. M123) :param guess: initial guess for the fitting, if none, some values are added automatically in order (see below) @@ -24,133 +40,185 @@ def fitccl( :param numfit_max: maximal value on x axis for numerical integration - if none is centre of gaussian plus 3 sigma :param constraints_min: min constranits value for fit :param constraints_max: max constranits value for fit - :return data dict with additional values - - order for guess, vary, constraints_min, constraints_max - [Gaussian centre, Gaussian sigma, Gaussian amplitude, Lorentzian centre, Lorentzian sigma, Lorentzian amplitude, background slope, background intercept] + order for guess, vary, constraints_min, constraints_max: + [Gaussian centre, Gaussian sigma, Gaussian amplitude, background slope, background intercept] examples: - guess = [None, None, 100, None, None, None, 0, None] - vary = [True, True, True, True, False, True, True, True] - constraints_min = [23, None, 50, None, None, None, 0, 0] - constraints_min = [80, None, 1000, None, None, None, 0, 100] + guess = [None, None, 100, 0, None] + vary = [True, True, True, True, True] + constraints_min = [23, None, 50, 0, 0] + constraints_min = [80, None, 1000, 0, 100] """ meas = data["Measurements"][keys] - if len(meas["peak_indexes"]) != 1: - print("NO PEAK or more than 1 peak") + if len(meas["peak_indexes"]) > 1: + # return in case of more than 1 peaks + print("More than 1 peak, measurement skipped") return x = list(meas["om"]) y = list(meas["Counts"]) - peak_index = meas["peak_indexes"] - peak_height = meas["peak_heights"] - print("before", constraints_min) - guess[0] = x[int(peak_index)] if guess[0] is None else guess[0] - guess[1] = 0.1 if guess[1] is None else guess[1] - guess[2] = float(peak_height / 10) if guess[2] is None else float(guess[2]) - guess[3] = x[int(peak_index)] if guess[3] is None else guess[3] - guess[4] = 2 * guess[1] if guess[4] is None else guess[4] - guess[5] = float(peak_height / 10) if guess[5] is None else float(guess[5]) - guess[6] = 0 if guess[6] is None else guess[6] - guess[7] = np.median(x) if guess[7] is None else guess[7] - constraints_min[0] = np.min(x) if constraints_min[0] is None else constraints_min[0] - constraints_min[3] = np.min(x) if constraints_min[3] is None else constraints_min[3] - constraints_max[0] = np.max(x) if constraints_max[0] is None else constraints_max[0] - constraints_max[3] = np.max(x) if constraints_max[3] is None else constraints_max[3] - print("key", keys) + # if the dictionaries were merged/substracted, takes the errors from them, if not, takes them as sqrt(y) + y_err = np.sqrt(y) if meas.get("sigma", None) is None else meas[keys].get("sigma") - print("after", constraints_min) + if len(meas["peak_indexes"]) == 0: + # Case for no peak, gaussian in centre, sigma as 20% of range + print("No peak") + peak_index = find_nearest(x, np.mean(x)) + guess[0] = x[int(peak_index)] + guess[1] = (x[-1] - x[0]) / 5 + guess[2] = 10 + guess[3] = 0 + guess[4] = np.mean(y) + constraints_min[2] = 0 - def find_nearest(array, value): - array = np.asarray(array) - idx = (np.abs(array - value)).argmin() - return idx + elif len(meas["peak_indexes"]) == 1: + # case for one peak, takse into account users guesses + print("one peak") + peak_index, peak_height = meas["peak_indexes"], meas["peak_heights"] + guess[0] = x[int(peak_index)] if guess[0] is None else guess[0] + guess[1] = 0.1 if guess[1] is None else guess[1] + guess[2] = float(peak_height / 10) if guess[2] is None else float(guess[2]) + guess[3] = 0 if guess[3] is None else guess[3] + guess[4] = np.median(x) if guess[4] is None else guess[4] + constraints_min[0] = np.min(x) if constraints_min[0] is None else constraints_min[0] + constraints_max[0] = np.max(x) if constraints_max[0] is None else constraints_max[0] + + centre = x[int(peak_index)] def gaussian(x, g_cen, g_width, g_amp): """1-d gaussian: gaussian(x, amp, cen, wid)""" - return (g_amp / (np.sqrt(2.0 * np.pi) * g_width)) * np.exp( + return (g_amp / (np.sqrt(2 * np.pi) * g_width)) * np.exp( -((x - g_cen) ** 2) / (2 * g_width ** 2) ) - def lorentzian(x, l_cen, l_width, l_amp): - """1d lorentzian""" - return (l_amp / (1 + ((1 * x - l_cen) / l_width) ** 2)) / (np.pi * l_width) - def background(x, slope, intercept): """background""" - return slope * x + intercept + return slope * (x - centre) + intercept - mod = Model(gaussian) + Model(lorentzian) + Model(background) + mod = Model(gaussian) + Model(background) params = Parameters() params.add_many( ("g_cen", x[int(peak_index)], bool(vary[0]), np.min(x), np.max(x), None, None), - ("g_width", guess[1], bool(vary[1]), constraints_min[1], constraints_max[1], None, None), - ("g_amp", guess[2], bool(vary[2]), constraints_min[2], constraints_max[2], None, None), - ("l_cen", guess[3], bool(vary[3]), np.min(x), np.max(x), None, None), - ("l_width", guess[4], bool(vary[4]), constraints_min[4], constraints_max[4], None, None), - ("l_amp", guess[5], bool(vary[5]), constraints_min[5], constraints_max[5], None, None), - ("slope", guess[6], bool(vary[6]), constraints_min[6], constraints_max[6], None, None), - ("intercept", guess[7], bool(vary[7]), constraints_min[7], constraints_max[7], None, None), + ( + "g_width", + guess[1], + bool(vary[1]), + constraints_min[1], + constraints_max[1], + None, + None, + ), + ( + "g_amp", + guess[2], + bool(vary[2]), + constraints_min[2], + constraints_max[2], + None, + None, + ), + ( + "slope", + guess[3], + bool(vary[3]), + constraints_min[3], + constraints_max[3], + None, + None, + ), + ( + "intercept", + guess[4], + bool(vary[4]), + constraints_min[4], + constraints_max[4], + None, + None, + ), ) - - result = mod.fit(y, params, x=x) - print("Chi-sqr: ", result.chisqr) - + # the weighted fit + result = mod.fit(y, params, weights=y_err, x=x, calc_covar=True) + # u.ufloat to work with uncertanities + fit_area = u.ufloat(result.params["g_amp"].value, result.params["g_amp"].stderr) comps = result.eval_components() - gauss_3sigmamin = find_nearest( - x, result.params["g_cen"].value - 3 * result.params["g_width"].value - ) - gauss_3sigmamax = find_nearest( - x, result.params["g_cen"].value + 3 * result.params["g_width"].value - ) - numfit_min = gauss_3sigmamin if numfit_min is None else find_nearest(x, numfit_min) - numfit_max = gauss_3sigmamax if numfit_max is None else find_nearest(x, numfit_max) - it = -1 + if len(meas["peak_indexes"]) == 0: + # for case of no peak, there is no reason to integrate, therefore fit and int are equal + int_area = fit_area - while numfit_max == numfit_min: - it = it + 1 - numfit_min = find_nearest( - x, result.params["g_cen"].value - 3 * (1 + it / 10) * result.params["g_width"].value + elif len(meas["peak_indexes"]) == 1: + gauss_3sigmamin = find_nearest( + x, result.params["g_cen"].value - 3 * result.params["g_width"].value ) - numfit_max = find_nearest( - x, result.params["g_cen"].value + 3 * (1 + it / 10) * result.params["g_width"].value + gauss_3sigmamax = find_nearest( + x, result.params["g_cen"].value + 3 * result.params["g_width"].value ) + numfit_min = gauss_3sigmamin if numfit_min is None else find_nearest(x, numfit_min) + numfit_max = gauss_3sigmamax if numfit_max is None else find_nearest(x, numfit_max) - if x[numfit_min] < np.min(x): - numfit_min = gauss_3sigmamin - print("Minimal integration value outside of x range") - elif x[numfit_min] >= x[numfit_max]: - numfit_min = gauss_3sigmamin - print("Minimal integration value higher than maximal") - else: - pass - if x[numfit_max] > np.max(x): - numfit_max = gauss_3sigmamax - print("Maximal integration value outside of x range") - elif x[numfit_max] <= x[numfit_min]: - numfit_max = gauss_3sigmamax - print("Maximal integration value lower than minimal") - else: - pass - print(result.params["g_width"].value) - print(result.params["g_cen"].value) - num_int_area = simps(y[numfit_min:numfit_max], x[numfit_min:numfit_max]) - num_int_bacground = integrate.quad( - background, - x[numfit_min], - x[numfit_max], - args=(result.params["slope"].value, result.params["intercept"].value), - ) + it = -1 + while numfit_max == numfit_min: + # in the case the peak is very thin and numerical integration would be on zero omega difference, finds closes values + it = it + 1 + numfit_min = find_nearest( + x, + result.params["g_cen"].value - 3 * (1 + it / 10) * result.params["g_width"].value, + ) + numfit_max = find_nearest( + x, + result.params["g_cen"].value + 3 * (1 + it / 10) * result.params["g_width"].value, + ) + + if x[numfit_min] < np.min(x): + # makes sure that the values supplied by user lay in the omega range + # can be ommited for users who know what they're doing + numfit_min = gauss_3sigmamin + print("Minimal integration value outside of x range") + elif x[numfit_min] >= x[numfit_max]: + numfit_min = gauss_3sigmamin + print("Minimal integration value higher than maximal") + else: + pass + if x[numfit_max] > np.max(x): + numfit_max = gauss_3sigmamax + print("Maximal integration value outside of x range") + elif x[numfit_max] <= x[numfit_min]: + numfit_max = gauss_3sigmamax + print("Maximal integration value lower than minimal") + else: + pass + + count_errors = create_uncertanities(y, y_err) + # create error vector for numerical integration propagation + num_int_area = simps(count_errors[numfit_min:numfit_max], x[numfit_min:numfit_max]) + slope_err = u.ufloat(result.params["slope"].value, result.params["slope"].stderr) + # pulls the nominal and error values from fit (slope) + intercept_err = u.ufloat( + result.params["intercept"].value, result.params["intercept"].stderr + ) + # pulls the nominal and error values from fit (intercept) + + background_errors = np.array([]) + for j in range(len(x[numfit_min:numfit_max])): + # creates nominal and error vector for numerical integration of background + bg = slope_err * (x[j] - centre) + intercept_err + background_errors = np.append(background_errors, bg) + + num_int_background = simps(background_errors, x[numfit_min:numfit_max]) + int_area = num_int_area - num_int_background d = {} for pars in result.params: d[str(pars)] = (result.params[str(pars)].value, result.params[str(pars)].vary) + print(result.fit_report()) + print((result.params["g_amp"].value - int_area.n) / result.params["g_amp"].value) d["export_fit"] = False - d["int_area"] = num_int_area - d["int_background"] = num_int_bacground + # ["export_fit"] = False if user wants num. int. value in comm/incomm, otherwise true + d["ratio"] = (result.params["g_amp"].value - int_area.n) / result.params["g_amp"].value + d["int_area"] = int_area + d["fit_area"] = fit_area d["full_report"] = result.fit_report() meas["fit"] = d From 4c0b195b7a294ee3077f16e1c64ec5625e57af68 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Mon, 21 Sep 2020 11:26:47 +0200 Subject: [PATCH 030/116] Update comm_export.py Added the fit value/ num integrated value choice --- pyzebra/comm_export.py | 80 +++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py index 27dc80a..a807415 100644 --- a/pyzebra/comm_export.py +++ b/pyzebra/comm_export.py @@ -30,46 +30,46 @@ def export_comm(data, path, lorentz=False): padding = [4, 6, 10, 8] with open(str(path + extension), "w") as out_file: - for meas_num, meas in data["Measurements"].items(): - print(meas_num) - if meas.get("fit") is None: - print("Measurement skipped - no fit value for:", meas_num) - continue + for keys in data["Measurements"]: + try: + meas_number_str = f"{keys[1:]:{align}{padding[0]}}" + h_str = f'{int(data["Measurements"][str(keys)]["h_index"]):{padding[1]}}' + k_str = f'{int(data["Measurements"][str(keys)]["k_index"]):{padding[1]}}' + l_str = f'{int(data["Measurements"][str(keys)]["l_index"]):{padding[1]}}' + if data["Measurements"][str(keys)]["fit"]["export_fit"] is True: + area = float(data["Measurements"][str(keys)]["fit"]["g_amp"].value) + sigma_str = f'{"{:8.2f}".format(float(data["Measurements"][str(keys)]["fit"]["g_amp"].stderr)):{align}{padding[2]}}' + else: + area = float(data["Measurements"][str(keys)]["fit"]["int_area"].n) + sigma_str = f'{"{:8.2f}".format(float(data["Measurements"][str(keys)]["fit"]["int_area"].s)):{align}{padding[2]}}' - meas_number_str = f"{meas_num[1:]:{align}{padding[0]}}" - h_str = f'{int(meas["h_index"]):{padding[1]}}' - k_str = f'{int(meas["k_index"]):{padding[1]}}' - l_str = f'{int(meas["l_index"]):{padding[1]}}' - if meas["fit"]["export_fit"] is True: - area = float(meas["fit"]["g_amp"][0]) + float(meas["fit"]["l_amp"][0]) - else: - area = float(meas["fit"]["int_area"]) - float(meas["fit"]["int_background"][0]) + if data["meta"]["zebra_mode"] == "bi": + int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], data["Measurements"][str(keys)]["twotheta_angle"])):{align}{padding[2]}}' + angle_str1 = f'{data["Measurements"][str(keys)]["twotheta_angle"]:{padding[3]}}' + angle_str2 = f'{data["Measurements"][str(keys)]["omega_angle"]:{padding[3]}}' + angle_str3 = f'{data["Measurements"][str(keys)]["chi_angle"]:{padding[3]}}' + angle_str4 = f'{data["Measurements"][str(keys)]["phi_angle"]:{padding[3]}}' + elif data["meta"]["zebra_mode"] == "nb": + int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], data["Measurements"][str(keys)]["gamma_angle"],data["Measurements"][str(keys)]["nu_angle"])):{align}{padding[2]}}' + angle_str1 = f'{data["Measurements"][str(keys)]["gamma_angle"]:{padding[3]}}' + angle_str2 = f'{data["Measurements"][str(keys)]["omega_angle"]:{padding[3]}}' + angle_str3 = f'{data["Measurements"][str(keys)]["nu_angle"]:{padding[3]}}' + angle_str4 = f'{data["Measurements"][str(keys)]["unkwn_angle"]:{padding[3]}}' - if data["meta"]["zebra_mode"] == "bi": - int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], meas["twotheta_angle"])):{align}{padding[2]}}' - angle_str1 = f'{meas["twotheta_angle"]:{padding[3]}}' - angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' - angle_str3 = f'{meas["chi_angle"]:{padding[3]}}' - angle_str4 = f'{meas["phi_angle"]:{padding[3]}}' - elif data["meta"]["zebra_mode"] == "nb": - int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], meas["gamma_angle"],meas["nu_angle"])):{align}{padding[2]}}' - angle_str1 = f'{meas["gamma_angle"]:{padding[3]}}' - angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' - angle_str3 = f'{meas["nu_angle"]:{padding[3]}}' - angle_str4 = f'{meas["unkwn_angle"]:{padding[3]}}' + line = ( + meas_number_str + + h_str + + l_str + + k_str + + int_str + + sigma_str + + angle_str1 + + angle_str2 + + angle_str3 + + angle_str4 + + "\n" + ) + out_file.write(line) - sigma_str = f'{"{:8.2f}".format(float(meas["fit"]["g_width"][0])):{align}{padding[2]}}' - line = ( - meas_number_str - + h_str - + l_str - + k_str - + int_str - + sigma_str - + angle_str1 - + angle_str2 - + angle_str3 - + angle_str4 - + "\n" - ) - out_file.write(line) + except KeyError: + print("Measurement skipped - no fit value for:", keys) From 5515db234b0001d4bf32d342a1ad4d478541b91a Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Mon, 21 Sep 2020 14:09:38 +0200 Subject: [PATCH 031/116] Update load_1D.py added the two unknown variables are temperature and field --- pyzebra/load_1D.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index cd1c75b..0d049a5 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -132,8 +132,8 @@ def parse_1D(fileobj, data_type): d["number_of_measurements"] = int(next_line.split()[0]) d["angle_step"] = float(next_line.split()[1]) d["monitor"] = float(next_line.split()[2]) - d["unkwn1"] = float(next_line.split()[3]) - d["unkwn2"] = float(next_line.split()[4]) + d["temperature"] = float(next_line.split()[3]) + d["mag_field"] = float(next_line.split()[4]) d["date"] = str(next_line.split()[5]) d["time"] = str(next_line.split()[6]) d["scan_type"] = str(next_line.split()[7]) From 8d397a819b70e96d441c8f6ac0bc04dff8384734 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Tue, 22 Sep 2020 08:47:31 +0200 Subject: [PATCH 032/116] Update fit2.py --- pyzebra/fit2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 49047a2..51ed99d 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -59,7 +59,7 @@ def fitccl( x = list(meas["om"]) y = list(meas["Counts"]) # if the dictionaries were merged/substracted, takes the errors from them, if not, takes them as sqrt(y) - y_err = np.sqrt(y) if meas.get("sigma", None) is None else meas[keys].get("sigma") + y_err = np.sqrt(y) if meas.get("sigma", None) is None else meas.get("sigma") if len(meas["peak_indexes"]) == 0: # Case for no peak, gaussian in centre, sigma as 20% of range From fd769106b7f36f1abac9ebfa47bde9188826d546 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 24 Sep 2020 10:53:42 +0200 Subject: [PATCH 033/116] Add a possiblity to select a measurement --- pyzebra/app/panel_1D_detector.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/pyzebra/app/panel_1D_detector.py b/pyzebra/app/panel_1D_detector.py index 52d72f0..1c33ecf 100644 --- a/pyzebra/app/panel_1D_detector.py +++ b/pyzebra/app/panel_1D_detector.py @@ -14,6 +14,7 @@ from bokeh.models import ( LinearAxis, Panel, Plot, + Select, Spacer, ) @@ -21,20 +22,22 @@ import pyzebra def create(): + det_data = {} + def upload_button_callback(_attr, _old, new): + nonlocal det_data with io.StringIO(base64.b64decode(new).decode()) as file: _, ext = os.path.splitext(upload_button.filename) - res = pyzebra.parse_1D(file, ext) - for _, meas in res["Measurements"].items(): - y = meas["Counts"] - x = list(range(len(y))) - break + det_data = pyzebra.parse_1D(file, ext) - plot_line_source.data.update(x=x, y=y) + meas_list = list(det_data["Measurements"].keys()) + meas_select.options = meas_list + meas_select.value = meas_list[0] upload_button = FileInput(accept=".ccl") upload_button.on_change("value", upload_button_callback) + # Main plot plot = Plot( x_range=DataRange1d(), y_range=DataRange1d(), @@ -52,7 +55,17 @@ def create(): plot_line_source = ColumnDataSource(dict(x=[0], y=[0])) plot.add_glyph(plot_line_source, Line(x="x", y="y", line_color="steelblue")) + # Measurement select + def meas_select_callback(_attr, _old, new): + y = det_data["Measurements"][new]["Counts"] + x = list(range(len(y))) + + plot_line_source.data.update(x=x, y=y) + + meas_select = Select() + meas_select.on_change("value", meas_select_callback) + upload_div = Div(text="Upload .ccl file:") - tab_layout = column(row(column(Spacer(height=5), upload_div), upload_button), plot) + tab_layout = column(row(column(Spacer(height=5), upload_div), upload_button, meas_select), plot) return Panel(child=tab_layout, title="1D Detector") From c977d3c58ec4ff5facd7bcfa721efd052f36effd Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 24 Sep 2020 15:20:48 +0200 Subject: [PATCH 034/116] Display basic peakfinding results --- pyzebra/app/panel_1D_detector.py | 39 ++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/pyzebra/app/panel_1D_detector.py b/pyzebra/app/panel_1D_detector.py index 1c33ecf..de18af1 100644 --- a/pyzebra/app/panel_1D_detector.py +++ b/pyzebra/app/panel_1D_detector.py @@ -5,6 +5,8 @@ import os from bokeh.layouts import column, row from bokeh.models import ( BasicTicker, + Button, + Circle, ColumnDataSource, DataRange1d, Div, @@ -37,6 +39,19 @@ def create(): upload_button = FileInput(accept=".ccl") upload_button.on_change("value", upload_button_callback) + def _update_plot(ind): + meas = det_data["Measurements"][ind] + y = meas["Counts"] + x = list(range(len(y))) + + plot_line_source.data.update(x=x, y=y) + + num_of_peaks = meas.get("num_of_peaks") + if num_of_peaks is not None and num_of_peaks > 0: + plot_circle_source.data.update(x=meas["peak_indexes"], y=meas["peak_heights"]) + else: + plot_circle_source.data.update(x=[], y=[]) + # Main plot plot = Plot( x_range=DataRange1d(), @@ -55,17 +70,31 @@ def create(): plot_line_source = ColumnDataSource(dict(x=[0], y=[0])) plot.add_glyph(plot_line_source, Line(x="x", y="y", line_color="steelblue")) + plot_circle_source = ColumnDataSource(dict(x=[], y=[])) + plot.add_glyph(plot_circle_source, Circle(x="x", y="y")) + # Measurement select def meas_select_callback(_attr, _old, new): - y = det_data["Measurements"][new]["Counts"] - x = list(range(len(y))) - - plot_line_source.data.update(x=x, y=y) + _update_plot(new) meas_select = Select() meas_select.on_change("value", meas_select_callback) + def process_button_callback(): + nonlocal det_data + for meas in det_data["Measurements"]: + det_data = pyzebra.ccl_findpeaks(det_data, meas) + + _update_plot(meas_select.value) + + process_button = Button(label="Process All") + process_button.on_click(process_button_callback) + upload_div = Div(text="Upload .ccl file:") - tab_layout = column(row(column(Spacer(height=5), upload_div), upload_button, meas_select), plot) + tab_layout = column( + row(column(Spacer(height=5), upload_div), upload_button, meas_select), + plot, + row(process_button), + ) return Panel(child=tab_layout, title="1D Detector") From 8bcbba5e318987274aeb46abb8467f0d7ee88a61 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 24 Sep 2020 15:35:53 +0200 Subject: [PATCH 035/116] Add smooth option as an example --- pyzebra/app/panel_1D_detector.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pyzebra/app/panel_1D_detector.py b/pyzebra/app/panel_1D_detector.py index de18af1..b2a7969 100644 --- a/pyzebra/app/panel_1D_detector.py +++ b/pyzebra/app/panel_1D_detector.py @@ -18,6 +18,7 @@ from bokeh.models import ( Plot, Select, Spacer, + Toggle, ) import pyzebra @@ -49,8 +50,10 @@ def create(): num_of_peaks = meas.get("num_of_peaks") if num_of_peaks is not None and num_of_peaks > 0: plot_circle_source.data.update(x=meas["peak_indexes"], y=meas["peak_heights"]) + plot_smooth_source.data.update(x=x, y=meas["smooth_peaks"]) else: plot_circle_source.data.update(x=[], y=[]) + plot_smooth_source.data.update(x=[], y=[]) # Main plot plot = Plot( @@ -70,6 +73,9 @@ def create(): plot_line_source = ColumnDataSource(dict(x=[0], y=[0])) plot.add_glyph(plot_line_source, Line(x="x", y="y", line_color="steelblue")) + plot_smooth_source = ColumnDataSource(dict(x=[0], y=[0])) + plot.add_glyph(plot_smooth_source, Line(x="x", y="y", line_color="red")) + plot_circle_source = ColumnDataSource(dict(x=[], y=[])) plot.add_glyph(plot_circle_source, Circle(x="x", y="y")) @@ -80,10 +86,12 @@ def create(): meas_select = Select() meas_select.on_change("value", meas_select_callback) + smooth_toggle = Toggle(label="Smooth curve") + def process_button_callback(): nonlocal det_data for meas in det_data["Measurements"]: - det_data = pyzebra.ccl_findpeaks(det_data, meas) + det_data = pyzebra.ccl_findpeaks(det_data, meas, smooth=smooth_toggle.active) _update_plot(meas_select.value) @@ -94,6 +102,7 @@ def create(): tab_layout = column( row(column(Spacer(height=5), upload_div), upload_button, meas_select), plot, + row(smooth_toggle), row(process_button), ) From 10a65613b9f1544c88eda32e603c1ddf3f4b5dd9 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 24 Sep 2020 16:07:58 +0200 Subject: [PATCH 036/116] Display fitting results --- pyzebra/app/panel_1D_detector.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/pyzebra/app/panel_1D_detector.py b/pyzebra/app/panel_1D_detector.py index b2a7969..d137eb2 100644 --- a/pyzebra/app/panel_1D_detector.py +++ b/pyzebra/app/panel_1D_detector.py @@ -18,6 +18,7 @@ from bokeh.models import ( Plot, Select, Spacer, + TextAreaInput, Toggle, ) @@ -55,6 +56,12 @@ def create(): plot_circle_source.data.update(x=[], y=[]) plot_smooth_source.data.update(x=[], y=[]) + fit = meas.get("fit") + if fit is not None: + fit_output_textinput.value = str(fit["full_report"]) + else: + fit_output_textinput.value = "" + # Main plot plot = Plot( x_range=DataRange1d(), @@ -88,20 +95,33 @@ def create(): smooth_toggle = Toggle(label="Smooth curve") + fit_output_textinput = TextAreaInput(title="Fit results:", width=600, height=400) + def process_button_callback(): nonlocal det_data for meas in det_data["Measurements"]: det_data = pyzebra.ccl_findpeaks(det_data, meas, smooth=smooth_toggle.active) + num_of_peaks = det_data["Measurements"][meas].get("num_of_peaks") + if num_of_peaks is not None and num_of_peaks == 1: + det_data = pyzebra.fitccl( + det_data, + meas, + guess=[None, None, None, None, None], + vary=[True, True, True, True, True], + constraints_min=[None, None, None, None, None], + constraints_max=[None, None, None, None, None], + ) + _update_plot(meas_select.value) - process_button = Button(label="Process All") + process_button = Button(label="Process All", button_type="primary") process_button.on_click(process_button_callback) upload_div = Div(text="Upload .ccl file:") tab_layout = column( row(column(Spacer(height=5), upload_div), upload_button, meas_select), - plot, + row(plot, fit_output_textinput), row(smooth_toggle), row(process_button), ) From 1bcecec76b0ef13e1ae24f0f7b75179b6e2de15a Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 24 Sep 2020 16:33:40 +0200 Subject: [PATCH 037/116] Display content of exported file --- pyzebra/app/panel_1D_detector.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/pyzebra/app/panel_1D_detector.py b/pyzebra/app/panel_1D_detector.py index d137eb2..c18ed5f 100644 --- a/pyzebra/app/panel_1D_detector.py +++ b/pyzebra/app/panel_1D_detector.py @@ -1,6 +1,7 @@ import base64 import io import os +import tempfile from bokeh.layouts import column, row from bokeh.models import ( @@ -67,7 +68,7 @@ def create(): x_range=DataRange1d(), y_range=DataRange1d(), plot_height=400, - plot_width=600, + plot_width=700, toolbar_location=None, ) @@ -118,12 +119,33 @@ def create(): process_button = Button(label="Process All", button_type="primary") process_button.on_click(process_button_callback) + output_textinput = TextAreaInput( + title="Content of comm/incomm file: (once fixed, Export button will save to user machine)", + width=600, + height=400, + ) + + def save_button_callback(): + if det_data["meta"]["indices"] == "hkl": + ext = ".comm" + elif det_data["meta"]["indices"] == "real": + ext = ".incomm" + + with tempfile.TemporaryDirectory() as temp_dir: + temp_file = temp_dir + "/temp" + pyzebra.export_comm(det_data, temp_file) + + with open(f"{temp_file}{ext}") as f: + output_textinput.value = f.read() + + save_button = Button(label="Export to .comm/.incomm file:") + save_button.on_click(save_button_callback) + upload_div = Div(text="Upload .ccl file:") tab_layout = column( row(column(Spacer(height=5), upload_div), upload_button, meas_select), - row(plot, fit_output_textinput), - row(smooth_toggle), - row(process_button), + row(plot, Spacer(width=30), fit_output_textinput, output_textinput), + row(column(smooth_toggle, process_button, save_button)), ) return Panel(child=tab_layout, title="1D Detector") From e5e824eb4ccd8a2e9a49b1b41174d0f6d30b630c Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 24 Sep 2020 16:46:35 +0200 Subject: [PATCH 038/116] Do not require matplotlib to be installed --- pyzebra/xtal.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyzebra/xtal.py b/pyzebra/xtal.py index fa2c3b0..ce4fcfd 100644 --- a/pyzebra/xtal.py +++ b/pyzebra/xtal.py @@ -1,11 +1,16 @@ import math import numpy as np -from matplotlib import pyplot as plt from scipy.optimize import curve_fit import pyzebra +try: + from matplotlib import pyplot as plt +except ImportError: + print("matplotlib is not available") + + def z4frgn(wave, ga, nu): """CALCULATES DIFFRACTION VECTOR IN LAB SYSTEM FROM GA AND NU From 52571a1aca48b4b4f5cf7e5b5ed6e2b39388dbdc Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 25 Sep 2020 09:12:00 +0200 Subject: [PATCH 039/116] Update fit2.py fit needs at least 3 points to work properly, this should not fail again in case of extremely slim peaks --- pyzebra/fit2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 51ed99d..fd55659 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -158,7 +158,7 @@ def fitccl( numfit_max = gauss_3sigmamax if numfit_max is None else find_nearest(x, numfit_max) it = -1 - while numfit_max == numfit_min: + while abs(numfit_max - numfit_min) < 3: # in the case the peak is very thin and numerical integration would be on zero omega difference, finds closes values it = it + 1 numfit_min = find_nearest( From a5022488911fb05a933e0caa9c31860840f0730d Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 25 Sep 2020 10:34:18 +0200 Subject: [PATCH 040/116] Simplify value references in export loop --- pyzebra/comm_export.py | 48 +++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py index a807415..a3fe998 100644 --- a/pyzebra/comm_export.py +++ b/pyzebra/comm_export.py @@ -20,7 +20,7 @@ def export_comm(data, path, lorentz=False): :arg data - data to export, is dict after peak fitting """ - + zebra_mode = data["meta"]["zebra_mode"] align = ">" if data["meta"]["indices"] == "hkl": extension = ".comm" @@ -30,31 +30,35 @@ def export_comm(data, path, lorentz=False): padding = [4, 6, 10, 8] with open(str(path + extension), "w") as out_file: - for keys in data["Measurements"]: + for keys, meas in data["Measurements"].items(): try: meas_number_str = f"{keys[1:]:{align}{padding[0]}}" - h_str = f'{int(data["Measurements"][str(keys)]["h_index"]):{padding[1]}}' - k_str = f'{int(data["Measurements"][str(keys)]["k_index"]):{padding[1]}}' - l_str = f'{int(data["Measurements"][str(keys)]["l_index"]):{padding[1]}}' - if data["Measurements"][str(keys)]["fit"]["export_fit"] is True: - area = float(data["Measurements"][str(keys)]["fit"]["g_amp"].value) - sigma_str = f'{"{:8.2f}".format(float(data["Measurements"][str(keys)]["fit"]["g_amp"].stderr)):{align}{padding[2]}}' + h_str = f'{int(meas["h_index"]):{padding[1]}}' + k_str = f'{int(meas["k_index"]):{padding[1]}}' + l_str = f'{int(meas["l_index"]):{padding[1]}}' + if meas["fit"]["export_fit"] is True: + area = float(meas["fit"]["g_amp"].value) + sigma_str = f'{"{:8.2f}".format(float(meas["fit"]["g_amp"].stderr)):{align}{padding[2]}}' else: - area = float(data["Measurements"][str(keys)]["fit"]["int_area"].n) - sigma_str = f'{"{:8.2f}".format(float(data["Measurements"][str(keys)]["fit"]["int_area"].s)):{align}{padding[2]}}' + area = float(meas["fit"]["int_area"].n) + sigma_str = ( + f'{"{:8.2f}".format(float(meas["fit"]["int_area"].s)):{align}{padding[2]}}' + ) - if data["meta"]["zebra_mode"] == "bi": - int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], data["Measurements"][str(keys)]["twotheta_angle"])):{align}{padding[2]}}' - angle_str1 = f'{data["Measurements"][str(keys)]["twotheta_angle"]:{padding[3]}}' - angle_str2 = f'{data["Measurements"][str(keys)]["omega_angle"]:{padding[3]}}' - angle_str3 = f'{data["Measurements"][str(keys)]["chi_angle"]:{padding[3]}}' - angle_str4 = f'{data["Measurements"][str(keys)]["phi_angle"]:{padding[3]}}' - elif data["meta"]["zebra_mode"] == "nb": - int_str = f'{"{:8.2f}".format(correction(area, lorentz, data["meta"]["zebra_mode"], data["Measurements"][str(keys)]["gamma_angle"],data["Measurements"][str(keys)]["nu_angle"])):{align}{padding[2]}}' - angle_str1 = f'{data["Measurements"][str(keys)]["gamma_angle"]:{padding[3]}}' - angle_str2 = f'{data["Measurements"][str(keys)]["omega_angle"]:{padding[3]}}' - angle_str3 = f'{data["Measurements"][str(keys)]["nu_angle"]:{padding[3]}}' - angle_str4 = f'{data["Measurements"][str(keys)]["unkwn_angle"]:{padding[3]}}' + if zebra_mode == "bi": + area = correction(area, lorentz, zebra_mode, meas["twotheta_angle"]) + int_str = f'{"{:8.2f}".format(area):{align}{padding[2]}}' + angle_str1 = f'{meas["twotheta_angle"]:{padding[3]}}' + angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' + angle_str3 = f'{meas["chi_angle"]:{padding[3]}}' + angle_str4 = f'{meas["phi_angle"]:{padding[3]}}' + elif zebra_mode == "nb": + area = correction(area, lorentz, zebra_mode, meas["gamma_angle"], meas["nu_angle"]) + int_str = f'{"{:8.2f}".format(area):{align}{padding[2]}}' + angle_str1 = f'{meas["gamma_angle"]:{padding[3]}}' + angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' + angle_str3 = f'{meas["nu_angle"]:{padding[3]}}' + angle_str4 = f'{meas["unkwn_angle"]:{padding[3]}}' line = ( meas_number_str From f79b4cbeeef9972c28ad1c395fd797a69a692269 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 25 Sep 2020 10:35:20 +0200 Subject: [PATCH 041/116] Check for "fit" key instead of handling KeyError --- pyzebra/comm_export.py | 92 +++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py index a3fe998..73d100d 100644 --- a/pyzebra/comm_export.py +++ b/pyzebra/comm_export.py @@ -31,49 +31,51 @@ def export_comm(data, path, lorentz=False): with open(str(path + extension), "w") as out_file: for keys, meas in data["Measurements"].items(): - try: - meas_number_str = f"{keys[1:]:{align}{padding[0]}}" - h_str = f'{int(meas["h_index"]):{padding[1]}}' - k_str = f'{int(meas["k_index"]):{padding[1]}}' - l_str = f'{int(meas["l_index"]):{padding[1]}}' - if meas["fit"]["export_fit"] is True: - area = float(meas["fit"]["g_amp"].value) - sigma_str = f'{"{:8.2f}".format(float(meas["fit"]["g_amp"].stderr)):{align}{padding[2]}}' - else: - area = float(meas["fit"]["int_area"].n) - sigma_str = ( - f'{"{:8.2f}".format(float(meas["fit"]["int_area"].s)):{align}{padding[2]}}' - ) - - if zebra_mode == "bi": - area = correction(area, lorentz, zebra_mode, meas["twotheta_angle"]) - int_str = f'{"{:8.2f}".format(area):{align}{padding[2]}}' - angle_str1 = f'{meas["twotheta_angle"]:{padding[3]}}' - angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' - angle_str3 = f'{meas["chi_angle"]:{padding[3]}}' - angle_str4 = f'{meas["phi_angle"]:{padding[3]}}' - elif zebra_mode == "nb": - area = correction(area, lorentz, zebra_mode, meas["gamma_angle"], meas["nu_angle"]) - int_str = f'{"{:8.2f}".format(area):{align}{padding[2]}}' - angle_str1 = f'{meas["gamma_angle"]:{padding[3]}}' - angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' - angle_str3 = f'{meas["nu_angle"]:{padding[3]}}' - angle_str4 = f'{meas["unkwn_angle"]:{padding[3]}}' - - line = ( - meas_number_str - + h_str - + l_str - + k_str - + int_str - + sigma_str - + angle_str1 - + angle_str2 - + angle_str3 - + angle_str4 - + "\n" - ) - out_file.write(line) - - except KeyError: + if "fit" not in meas: print("Measurement skipped - no fit value for:", keys) + continue + + meas_number_str = f"{keys[1:]:{align}{padding[0]}}" + h_str = f'{int(meas["h_index"]):{padding[1]}}' + k_str = f'{int(meas["k_index"]):{padding[1]}}' + l_str = f'{int(meas["l_index"]):{padding[1]}}' + if meas["fit"]["export_fit"] is True: + area = float(meas["fit"]["g_amp"].value) + sigma_str = ( + f'{"{:8.2f}".format(float(meas["fit"]["g_amp"].stderr)):{align}{padding[2]}}' + ) + else: + area = float(meas["fit"]["int_area"].n) + sigma_str = ( + f'{"{:8.2f}".format(float(meas["fit"]["int_area"].s)):{align}{padding[2]}}' + ) + + if zebra_mode == "bi": + area = correction(area, lorentz, zebra_mode, meas["twotheta_angle"]) + int_str = f'{"{:8.2f}".format(area):{align}{padding[2]}}' + angle_str1 = f'{meas["twotheta_angle"]:{padding[3]}}' + angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' + angle_str3 = f'{meas["chi_angle"]:{padding[3]}}' + angle_str4 = f'{meas["phi_angle"]:{padding[3]}}' + elif zebra_mode == "nb": + area = correction(area, lorentz, zebra_mode, meas["gamma_angle"], meas["nu_angle"]) + int_str = f'{"{:8.2f}".format(area):{align}{padding[2]}}' + angle_str1 = f'{meas["gamma_angle"]:{padding[3]}}' + angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' + angle_str3 = f'{meas["nu_angle"]:{padding[3]}}' + angle_str4 = f'{meas["unkwn_angle"]:{padding[3]}}' + + line = ( + meas_number_str + + h_str + + l_str + + k_str + + int_str + + sigma_str + + angle_str1 + + angle_str2 + + angle_str3 + + angle_str4 + + "\n" + ) + out_file.write(line) From 665de61c8ca71349a2aee2d86ed1e2d3311c611a Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 25 Sep 2020 13:23:49 +0200 Subject: [PATCH 042/116] functions to merge, scan, subtract measurements --- ccl_dict_operation_v1.py | 418 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 418 insertions(+) create mode 100644 ccl_dict_operation_v1.py diff --git a/ccl_dict_operation_v1.py b/ccl_dict_operation_v1.py new file mode 100644 index 0000000..ee79c89 --- /dev/null +++ b/ccl_dict_operation_v1.py @@ -0,0 +1,418 @@ +import numpy as np +import uncertainties as u +from fitvol2 import create_uncertanities + + +def scan_dict(dict): + """scans dictionary for duplicate hkl indexes + :arg dict : dictionary to scan + :return dictionary with matching scans, if there are none, the dict is empty + note: can be checked by "not d", true if empty + """ + d = {} + dupl = 0 + for i in dict["Measurements"]: + for j in dict["Measurements"]: + if dict["Measurements"][str(i)] != dict["Measurements"][str(j)]: + itup = ( + dict["Measurements"][str(i)]["h_index"], + dict["Measurements"][str(i)]["k_index"], + dict["Measurements"][str(i)]["l_index"], + ) + jtup = ( + dict["Measurements"][str(j)]["h_index"], + dict["Measurements"][str(j)]["k_index"], + dict["Measurements"][str(j)]["l_index"], + ) + if itup != jtup: + pass + else: + if str(itup) not in d: + d[str(itup)] = (i, j) + else: + dupl = dupl + 1 + d[str(itup) + "_" + str(dupl)] = (i, j) + else: + continue + return d + + +def compare_hkl(dict1, dict2): + """Compares two dictionaries based on hkl indexes and return dictionary with str(h k l) as key and tuple with + keys to same measurement in dict1 and dict2 + :arg dict1 : first dictionary + :arg dict2 : second dictionary + :return d : dict with matches + example of one key: '0.0 0.0 -1.0 : ('M1', 'M9')' meaning that 001 hkl measurement is M1 in first dict and M9 in second""" + d = {} + dupl = 0 + for keys in dict1["Measurements"]: + for key in dict2["Measurements"]: + if ( + dict1["Measurements"][str(keys)]["h_index"] + == dict2["Measurements"][str(key)]["h_index"] + and dict1["Measurements"][str(keys)]["k_index"] + == dict2["Measurements"][str(key)]["k_index"] + and dict1["Measurements"][str(keys)]["l_index"] + == dict2["Measurements"][str(key)]["l_index"] + ): + + if ( + str( + ( + str(dict1["Measurements"][str(keys)]["h_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["k_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["l_index"]) + ) + ) + not in d + ): + d[ + str( + str(dict1["Measurements"][str(keys)]["h_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["k_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["l_index"]) + ) + ] = (str(keys), str(key)) + else: + dupl = dupl + 1 + d[ + str( + str(dict1["Measurements"][str(keys)]["h_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["k_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["l_index"]) + + "_dupl" + + str(dupl) + ) + ] = (str(keys), str(key)) + else: + continue + + return d + + +def create_tuples(x, y, y_err): + """creates tuples for sorting and merginng of the data + Counts need to be normalized to monitor before""" + t = list() + for i in range(len(x)): + tup = (x[i], y[i], y_err) + t.append(tup) + return t + + +def merge(dict1, dict2, keys, auto=True, monitor=100000): + """merges the two tuples and sorts them, if om value is same, Counts value is average + averaging is propagated into sigma if dict1 == dict2, key[1] is deleted after merging + :arg dict1 : dictionary to which measurement will be merged + :arg dict2 : dictionary from which measurement will be merged + :arg keys : tuple with key to dict1 and dict2 + :arg auto : if true, when monitors are same, does not change it, if flase, takes monitor always + :arg monitor : final monitor after merging + note: dict1 and dict2 can be same dict + :return dict1 with merged scan""" + if auto: + if dict1["Measurements"][keys[0]]["monitor"] == dict2["Measurements"][keys[1]]["monitor"]: + monitor = dict1["Measurements"][keys[0]]["monitor"] + + # load om and Counts + x1, x2 = dict1["Measurements"][keys[0]]["om"], dict2["Measurements"][keys[1]]["om"] + y1, y2 = np.array(dict1["Measurements"][keys[0]]["Counts"]), np.array( + dict2["Measurements"][keys[1]]["Counts"] + ) + # normalize y to monitors + cor_y1 = (y1 / dict1["Measurements"][keys[0]]["monitor"]) * monitor + cor_y2 = (y2 / dict2["Measurements"][keys[0]]["monitor"]) * monitor + # check if sigma errors for y exist, otherwise create them as sqrt(y) + y_err1 = ( + np.sqrt(cor_y1) + if "sigma" not in dict1["Measurements"][keys[0]] + else np.array(dict1["Measurements"][keys[0]]["sigma"]) + * np.sqrt(monitor / dict1["Measurements"][keys[0]]["monitor"]) + ) + y_err2 = ( + np.sqrt(cor_y2) + if "sigma" not in dict2["Measurements"][keys[1]] + else np.array(dict2["Measurements"][keys[1]]["sigma"]) + * np.sqrt(monitor / dict2["Measurements"][keys[1]]["monitor"]) + ) + # creates touples (om, Counts, sigma) for sorting and further processing + tuple_list = create_tuples(x1, cor_y1, y_err1) + create_tuples(x2, cor_y2, y_err2) + # Sort the list on om and add 0 0 0 tuple to the last postion + sorted_t = sorted(tuple_list, key=lambda tup: tup[0]) + sorted_t.append((0, 0, 0)) + om, Counts, sigma = [], [], [] + seen = list() + for i in range(len(sorted_t) - 1): + if sorted_t[i][0] not in seen: + if sorted_t[i][0] != sorted_t[i + 1][0]: + om = np.append(om, sorted_t[i][0]) + Counts = np.append(Counts, sorted_t[i][1]) + sigma = np.append(sigma, sorted_t[i][2]) + else: + om = np.append(om, sorted_t[i][0]) + counts1, counts2 = sorted_t[i][1], sorted_t[i + 1][1] + sigma1, sigma2 = sorted_t[i][2], sorted_t[i + 1][2] + count_err1 = u.ufloat(counts1, sigma1) + count_err2 = u.ufloat(counts2, sigma2) + avg = (count_err1 + count_err2) / 2 + Counts = np.append(Counts, avg.n) + sigma = np.append(sigma, avg.s) + seen.append(sorted_t[i][0]) + else: + continue + + if dict1 == dict2: + del dict1["Measurements"][keys[1]] + + dict1["Measurements"][keys[0]]["om"] = om + dict1["Measurements"][keys[0]]["Counts"] = Counts + dict1["Measurements"][keys[0]]["sigma"] = sigma + return dict1 + + +def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): + """Substracts two measurement (measurement key2 from dict2 from measurent key1 in dict1), expects om to be same + :arg dict1 : dictionary to which measurement will be merged + :arg dict2 : dictionary from which measurement will be merged + :arg keys : tuple with key to dict1 and dict2 + :arg auto : if true, when monitors are same, does not change it, if flase, takes monitor always + :arg monitor : final monitor after merging + :returns d : dict1 with substracted Counts from dict2 and sigma that comes from the substraction""" + + if len(dict1["Measurements"][keys[0]]["om"]) != len(dict2["Measurements"][keys[1]]["om"]): + raise ValueError("Omegas have different lengths, cannot be substracted") + + if auto: + if dict1["Measurements"][keys[0]]["monitor"] == dict2["Measurements"][keys[1]]["monitor"]: + monitor = dict1["Measurements"][keys[0]]["monitor"] + + monitor_ratio_prim = monitor / dict1["Measurements"][str(keys[0])]["monitor"] + monitor_ratio_sec = monitor / dict2["Measurements"][str(keys[1])]["monitor"] + y1 = np.array(dict1["Measurements"][str(keys[0])]["Counts"]) * monitor_ratio_prim + y2 = np.array(dict2["Measurements"][str(keys[1])]["Counts"]) * monitor_ratio_sec + y_err1 = ( + np.sqrt(y1) + if "sigma" not in dict1["Measurements"][keys[0]] + else dict1["Measurements"][keys[0]]["sigma"] + * np.sqrt(monitor / dict1["Measurements"][keys[0]]["monitor"]) + ) + y_err2 = ( + np.sqrt(y2) + if "sigma" not in dict2["Measurements"][keys[1]] + else dict2["Measurements"][keys[1]]["sigma"] + * np.sqrt(monitor / dict2["Measurements"][keys[1]]["monitor"]) + ) + dict1_count_err = create_uncertanities(y1, y_err1) + dict2_count_err = create_uncertanities(y2, y_err2) + if np.average(y1) > np.average(y2): + res = np.subtract(dict1_count_err, dict2_count_err) + else: + res = np.subtract(dict2_count_err, dict1_count_err) + res_nom = [] + res_err = [] + for k in range(len(res)): + res_nom = np.append(res_nom, res[k].n) + res_err = np.append(res_err, res[k].s) + dict1["Measurements"][str(keys[0])]["Counts"] = res_nom + dict1["Measurements"][str(keys[0])]["sigma"] = res_err + + +def compare_dict(dict1, dict2): + """takes two ccl dictionaries and compare different values for each key + :arg dict1 : dictionary 1 (ccl) + :arg dict2 : dictionary 2 (ccl) + :returns warning : dictionary with keys from primary files (if they differ) with + information of how many measurement differ and which ones differ + :returns report_string string comparing all different values respecively of measurements""" + + if dict1["file_type"] != dict2["file_type"]: + print("select two dicts") + return + S = [] + conflicts = {} + warnings = {} + + comp = compare_hkl(dict1, dict2) + d1 = scan_dict(dict1) + d2 = scan_dict(dict2) + if not d1: + S.append("There are no duplicates in %s (dict1) \n" % dict1["meta"]["original_filename"]) + else: + S.append( + "There are %d duplicates in %s (dict1) \n" + % (len(d1), dict1["meta"]["original_filename"]) + ) + warnings["Duplicates in dict1"] = list() + for keys in d1: + S.append("Measurements %s with hkl %s \n" % (d1[keys], keys)) + warnings["Duplicates in dict1"].append(d1[keys]) + if not d2: + S.append("There are no duplicates in %s (dict2) \n" % dict2["meta"]["original_filename"]) + else: + S.append( + "There are %d duplicates in %s (dict2) \n" + % (len(d2), dict2["meta"]["original_filename"]) + ) + warnings["Duplicates in dict2"] = list() + for keys in d2: + S.append("Measurements %s with hkl %s \n" % (d2[keys], keys)) + warnings["Duplicates in dict2"].append(d2[keys]) + + # compare meta + S.append("Different values in meta: \n") + different_meta = { + k: dict1["meta"][k] + for k in dict1["meta"] + if k in dict2["meta"] and dict1["meta"][k] != dict2["meta"][k] + } + + for keys in different_meta: + if keys not in conflicts: + conflicts[keys] = 1 + else: + conflicts[keys] = conflicts[keys] + 1 + + S.append(" Different values in %s \n" % str(keys)) + S.append(" dict1: %s \n" % str(dict1["meta"][str(keys)])) + S.append(" dict2: %s \n" % str(dict2["meta"][str(keys)])) + + # compare Measurements + S.append( + "Number of measurements in %s = %s \n" + % (dict1["meta"]["original_filename"], len(dict1["Measurements"])) + ) + S.append( + "Number of measurements in %s = %s \n" + % (dict2["meta"]["original_filename"], len(dict2["Measurements"])) + ) + S.append("Different values in Measurements:\n") + select_set = ["om", "Counts", "sigma"] + exlude_set = ["time", "Counts"] + + for keys1 in comp: + for key2 in dict1["Measurements"][str(comp[str(keys1)][0])]: + if key2 in exlude_set: + continue + if key2 not in select_set: + if ( + dict1["Measurements"][comp[str(keys1)][0]][str(key2)] + != dict2["Measurements"][str(comp[str(keys1)][0])][str(key2)] + ): + S.append( + "Measurement value " + "%s" + ", with hkl %s differs in meausrements %s and %s \n" + % (key2, keys1, comp[str(keys1)][0], comp[str(keys1)][1]) + ) + S.append( + " dict1: %s \n" + % str(dict1["Measurements"][comp[str(keys1)][0]][str(key2)]) + ) + S.append( + " dict2: %s \n" + % str(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) + ) + if key2 not in conflicts: + conflicts[key2] = {} + conflicts[key2]["amount"] = 1 + conflicts[key2]["measurements"] = str(comp[str(keys1)][0]) + else: + conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 + conflicts[key2]["measurements"] = ( + conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)][0])) + ) + else: + comparison = list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)]) == list( + dict2["Measurements"][comp[str(keys1)][1]][str(key2)] + ) + if len(list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)])) != len( + list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) + ): + if str("different length of %s" % key2) not in warnings: + warnings[str("different length of %s" % key2)] = list() + warnings[str("different length of %s" % key2)].append( + (str(comp[keys1][0]), str(comp[keys1][1])) + ) + else: + warnings[str("different length of %s" % key2)].append( + (str(comp[keys1][0]), str(comp[keys1][1])) + ) + if not comparison: + S.append( + "Measurement value " + "%s" + " differs in measurement %s and %s \n" + % (key2, comp[str(keys1)][0], comp[str(keys1)][1]) + ) + S.append( + " dict1: %s \n" + % str(list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)])) + ) + S.append( + " dict2: %s \n" + % str(list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)])) + ) + if key2 not in conflicts: + conflicts[key2] = {} + conflicts[key2]["amount"] = 1 + conflicts[key2]["measurements"] = str(comp[str(keys1)][0]) + else: + conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 + conflicts[key2]["measurements"] = ( + conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)][1])) + ) + + for keys in conflicts: + try: + conflicts[str(keys)]["measurements"] = conflicts[str(keys)]["measurements"].split(" ") + except: + continue + report_string = "".join(S) + return warnings, conflicts, report_string + + +def guess_next(dict1, dict2, comp): + """iterates thorough the scans and tries to decide if the scans should be substracted or merged""" + threshold = 0.05 + for keys in comp: + if ( + abs( + ( + dict1["Measurements"][str(comp[keys][0])]["temperature"] + - dict2["Measurements"][str(comp[keys][1])]["temperature"] + ) + / dict2["Measurements"][str(comp[keys][1])]["temperature"] + ) + < threshold + and abs( + ( + dict1["Measurements"][str(comp[keys][0])]["mag_field"] + - dict2["Measurements"][str(comp[keys][1])]["mag_field"] + ) + / dict2["Measurements"][str(comp[keys][1])]["mag_field"] + ) + < threshold + ): + comp[keys] = comp[keys] + tuple("m") + else: + comp[keys] = comp[keys] + tuple("s") + + return comp + + +def process_dict(dict1, dict2, comp): + """substracts or merges scans, guess_next function must run first """ + for keys in comp: + if comp[keys][2] == "s": + substract_measurement(dict1, dict2, comp[keys]) + elif comp[keys][2] == "m": + merge(dict1, dict2, comp[keys]) + + return dict1 From 54a58dd4cadb19e6600763d6a71b4b9f23d9f0dc Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 25 Sep 2020 15:40:37 +0200 Subject: [PATCH 043/116] Update ccl_dict_operation_v1.py --- ccl_dict_operation_v1.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ccl_dict_operation_v1.py b/ccl_dict_operation_v1.py index ee79c89..1415321 100644 --- a/ccl_dict_operation_v1.py +++ b/ccl_dict_operation_v1.py @@ -211,10 +211,8 @@ def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): ) dict1_count_err = create_uncertanities(y1, y_err1) dict2_count_err = create_uncertanities(y2, y_err2) - if np.average(y1) > np.average(y2): - res = np.subtract(dict1_count_err, dict2_count_err) - else: - res = np.subtract(dict2_count_err, dict1_count_err) + + res = np.subtract(dict1_count_err, dict2_count_err) res_nom = [] res_err = [] for k in range(len(res)): From 0c9e906ba4c0a780110aa9d073c145b59052f1b2 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 25 Sep 2020 16:19:10 +0200 Subject: [PATCH 044/116] Visualize fit Copy and paste into fit2 righ after: num_int_background = simps(background_errors, x[numfit_min:numfit_max]) int_area = num_int_area-num_int_background ca line 180 --- pyzebra/visualize peak fitter.py | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 pyzebra/visualize peak fitter.py diff --git a/pyzebra/visualize peak fitter.py b/pyzebra/visualize peak fitter.py new file mode 100644 index 0000000..fc4b034 --- /dev/null +++ b/pyzebra/visualize peak fitter.py @@ -0,0 +1,39 @@ +plt.figure(figsize=(20, 10)) +plt.plot(x, y, "b-", label="Original data") +plt.plot(x, comps["gaussian"], "r--", label="Gaussian component") +plt.fill_between(x, comps["gaussian"], facecolor="red", alpha=0.4) +plt.plot(x, comps["background"], "g--", label="Line component") +plt.fill_between(x, comps["background"], facecolor="green", alpha=0.4) + +plt.fill_between( + x[numfit_min:numfit_max], + y[numfit_min:numfit_max], + facecolor="yellow", + alpha=0.4, + label="Integrated area", ) + +plt.plot(x, result.best_fit, "k-", label="Best fit") +plt.title( + "%s \n Gaussian: centre = %9.4f, sigma = %9.4f, area = %9.4f \n" + "background: slope = %9.4f, intercept = %9.4f \n" + "Int. area = %9.4f +/- %9.4f \n" + "fit area = %9.4f +/- %9.4f \n" + "ratio((fit-int)/fit) = %9.4f" + % ( + keys, + result.params["g_cen"].value, + result.params["g_width"].value, + result.params["g_amp"].value, + result.params["slope"].value, + result.params["intercept"].value, + int_area.n, + int_area.s, + result.params["g_amp"].value, + result.params["g_amp"].stderr, + (result.params["g_amp"].value - int_area.n) / result.params["g_amp"].value + + ) +) + +plt.legend(loc="best") +plt.show() \ No newline at end of file From 188f6d51632efae6cbb57370a8f540aa33f3c3c3 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 25 Sep 2020 16:22:08 +0200 Subject: [PATCH 045/116] Add files via upload Reuploaded from wrong folder --- pyzebra/ccl_dict_operation.py | 418 ++++++++++++++++++++++++++++++++++ 1 file changed, 418 insertions(+) create mode 100644 pyzebra/ccl_dict_operation.py diff --git a/pyzebra/ccl_dict_operation.py b/pyzebra/ccl_dict_operation.py new file mode 100644 index 0000000..ee79c89 --- /dev/null +++ b/pyzebra/ccl_dict_operation.py @@ -0,0 +1,418 @@ +import numpy as np +import uncertainties as u +from fitvol2 import create_uncertanities + + +def scan_dict(dict): + """scans dictionary for duplicate hkl indexes + :arg dict : dictionary to scan + :return dictionary with matching scans, if there are none, the dict is empty + note: can be checked by "not d", true if empty + """ + d = {} + dupl = 0 + for i in dict["Measurements"]: + for j in dict["Measurements"]: + if dict["Measurements"][str(i)] != dict["Measurements"][str(j)]: + itup = ( + dict["Measurements"][str(i)]["h_index"], + dict["Measurements"][str(i)]["k_index"], + dict["Measurements"][str(i)]["l_index"], + ) + jtup = ( + dict["Measurements"][str(j)]["h_index"], + dict["Measurements"][str(j)]["k_index"], + dict["Measurements"][str(j)]["l_index"], + ) + if itup != jtup: + pass + else: + if str(itup) not in d: + d[str(itup)] = (i, j) + else: + dupl = dupl + 1 + d[str(itup) + "_" + str(dupl)] = (i, j) + else: + continue + return d + + +def compare_hkl(dict1, dict2): + """Compares two dictionaries based on hkl indexes and return dictionary with str(h k l) as key and tuple with + keys to same measurement in dict1 and dict2 + :arg dict1 : first dictionary + :arg dict2 : second dictionary + :return d : dict with matches + example of one key: '0.0 0.0 -1.0 : ('M1', 'M9')' meaning that 001 hkl measurement is M1 in first dict and M9 in second""" + d = {} + dupl = 0 + for keys in dict1["Measurements"]: + for key in dict2["Measurements"]: + if ( + dict1["Measurements"][str(keys)]["h_index"] + == dict2["Measurements"][str(key)]["h_index"] + and dict1["Measurements"][str(keys)]["k_index"] + == dict2["Measurements"][str(key)]["k_index"] + and dict1["Measurements"][str(keys)]["l_index"] + == dict2["Measurements"][str(key)]["l_index"] + ): + + if ( + str( + ( + str(dict1["Measurements"][str(keys)]["h_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["k_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["l_index"]) + ) + ) + not in d + ): + d[ + str( + str(dict1["Measurements"][str(keys)]["h_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["k_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["l_index"]) + ) + ] = (str(keys), str(key)) + else: + dupl = dupl + 1 + d[ + str( + str(dict1["Measurements"][str(keys)]["h_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["k_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["l_index"]) + + "_dupl" + + str(dupl) + ) + ] = (str(keys), str(key)) + else: + continue + + return d + + +def create_tuples(x, y, y_err): + """creates tuples for sorting and merginng of the data + Counts need to be normalized to monitor before""" + t = list() + for i in range(len(x)): + tup = (x[i], y[i], y_err) + t.append(tup) + return t + + +def merge(dict1, dict2, keys, auto=True, monitor=100000): + """merges the two tuples and sorts them, if om value is same, Counts value is average + averaging is propagated into sigma if dict1 == dict2, key[1] is deleted after merging + :arg dict1 : dictionary to which measurement will be merged + :arg dict2 : dictionary from which measurement will be merged + :arg keys : tuple with key to dict1 and dict2 + :arg auto : if true, when monitors are same, does not change it, if flase, takes monitor always + :arg monitor : final monitor after merging + note: dict1 and dict2 can be same dict + :return dict1 with merged scan""" + if auto: + if dict1["Measurements"][keys[0]]["monitor"] == dict2["Measurements"][keys[1]]["monitor"]: + monitor = dict1["Measurements"][keys[0]]["monitor"] + + # load om and Counts + x1, x2 = dict1["Measurements"][keys[0]]["om"], dict2["Measurements"][keys[1]]["om"] + y1, y2 = np.array(dict1["Measurements"][keys[0]]["Counts"]), np.array( + dict2["Measurements"][keys[1]]["Counts"] + ) + # normalize y to monitors + cor_y1 = (y1 / dict1["Measurements"][keys[0]]["monitor"]) * monitor + cor_y2 = (y2 / dict2["Measurements"][keys[0]]["monitor"]) * monitor + # check if sigma errors for y exist, otherwise create them as sqrt(y) + y_err1 = ( + np.sqrt(cor_y1) + if "sigma" not in dict1["Measurements"][keys[0]] + else np.array(dict1["Measurements"][keys[0]]["sigma"]) + * np.sqrt(monitor / dict1["Measurements"][keys[0]]["monitor"]) + ) + y_err2 = ( + np.sqrt(cor_y2) + if "sigma" not in dict2["Measurements"][keys[1]] + else np.array(dict2["Measurements"][keys[1]]["sigma"]) + * np.sqrt(monitor / dict2["Measurements"][keys[1]]["monitor"]) + ) + # creates touples (om, Counts, sigma) for sorting and further processing + tuple_list = create_tuples(x1, cor_y1, y_err1) + create_tuples(x2, cor_y2, y_err2) + # Sort the list on om and add 0 0 0 tuple to the last postion + sorted_t = sorted(tuple_list, key=lambda tup: tup[0]) + sorted_t.append((0, 0, 0)) + om, Counts, sigma = [], [], [] + seen = list() + for i in range(len(sorted_t) - 1): + if sorted_t[i][0] not in seen: + if sorted_t[i][0] != sorted_t[i + 1][0]: + om = np.append(om, sorted_t[i][0]) + Counts = np.append(Counts, sorted_t[i][1]) + sigma = np.append(sigma, sorted_t[i][2]) + else: + om = np.append(om, sorted_t[i][0]) + counts1, counts2 = sorted_t[i][1], sorted_t[i + 1][1] + sigma1, sigma2 = sorted_t[i][2], sorted_t[i + 1][2] + count_err1 = u.ufloat(counts1, sigma1) + count_err2 = u.ufloat(counts2, sigma2) + avg = (count_err1 + count_err2) / 2 + Counts = np.append(Counts, avg.n) + sigma = np.append(sigma, avg.s) + seen.append(sorted_t[i][0]) + else: + continue + + if dict1 == dict2: + del dict1["Measurements"][keys[1]] + + dict1["Measurements"][keys[0]]["om"] = om + dict1["Measurements"][keys[0]]["Counts"] = Counts + dict1["Measurements"][keys[0]]["sigma"] = sigma + return dict1 + + +def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): + """Substracts two measurement (measurement key2 from dict2 from measurent key1 in dict1), expects om to be same + :arg dict1 : dictionary to which measurement will be merged + :arg dict2 : dictionary from which measurement will be merged + :arg keys : tuple with key to dict1 and dict2 + :arg auto : if true, when monitors are same, does not change it, if flase, takes monitor always + :arg monitor : final monitor after merging + :returns d : dict1 with substracted Counts from dict2 and sigma that comes from the substraction""" + + if len(dict1["Measurements"][keys[0]]["om"]) != len(dict2["Measurements"][keys[1]]["om"]): + raise ValueError("Omegas have different lengths, cannot be substracted") + + if auto: + if dict1["Measurements"][keys[0]]["monitor"] == dict2["Measurements"][keys[1]]["monitor"]: + monitor = dict1["Measurements"][keys[0]]["monitor"] + + monitor_ratio_prim = monitor / dict1["Measurements"][str(keys[0])]["monitor"] + monitor_ratio_sec = monitor / dict2["Measurements"][str(keys[1])]["monitor"] + y1 = np.array(dict1["Measurements"][str(keys[0])]["Counts"]) * monitor_ratio_prim + y2 = np.array(dict2["Measurements"][str(keys[1])]["Counts"]) * monitor_ratio_sec + y_err1 = ( + np.sqrt(y1) + if "sigma" not in dict1["Measurements"][keys[0]] + else dict1["Measurements"][keys[0]]["sigma"] + * np.sqrt(monitor / dict1["Measurements"][keys[0]]["monitor"]) + ) + y_err2 = ( + np.sqrt(y2) + if "sigma" not in dict2["Measurements"][keys[1]] + else dict2["Measurements"][keys[1]]["sigma"] + * np.sqrt(monitor / dict2["Measurements"][keys[1]]["monitor"]) + ) + dict1_count_err = create_uncertanities(y1, y_err1) + dict2_count_err = create_uncertanities(y2, y_err2) + if np.average(y1) > np.average(y2): + res = np.subtract(dict1_count_err, dict2_count_err) + else: + res = np.subtract(dict2_count_err, dict1_count_err) + res_nom = [] + res_err = [] + for k in range(len(res)): + res_nom = np.append(res_nom, res[k].n) + res_err = np.append(res_err, res[k].s) + dict1["Measurements"][str(keys[0])]["Counts"] = res_nom + dict1["Measurements"][str(keys[0])]["sigma"] = res_err + + +def compare_dict(dict1, dict2): + """takes two ccl dictionaries and compare different values for each key + :arg dict1 : dictionary 1 (ccl) + :arg dict2 : dictionary 2 (ccl) + :returns warning : dictionary with keys from primary files (if they differ) with + information of how many measurement differ and which ones differ + :returns report_string string comparing all different values respecively of measurements""" + + if dict1["file_type"] != dict2["file_type"]: + print("select two dicts") + return + S = [] + conflicts = {} + warnings = {} + + comp = compare_hkl(dict1, dict2) + d1 = scan_dict(dict1) + d2 = scan_dict(dict2) + if not d1: + S.append("There are no duplicates in %s (dict1) \n" % dict1["meta"]["original_filename"]) + else: + S.append( + "There are %d duplicates in %s (dict1) \n" + % (len(d1), dict1["meta"]["original_filename"]) + ) + warnings["Duplicates in dict1"] = list() + for keys in d1: + S.append("Measurements %s with hkl %s \n" % (d1[keys], keys)) + warnings["Duplicates in dict1"].append(d1[keys]) + if not d2: + S.append("There are no duplicates in %s (dict2) \n" % dict2["meta"]["original_filename"]) + else: + S.append( + "There are %d duplicates in %s (dict2) \n" + % (len(d2), dict2["meta"]["original_filename"]) + ) + warnings["Duplicates in dict2"] = list() + for keys in d2: + S.append("Measurements %s with hkl %s \n" % (d2[keys], keys)) + warnings["Duplicates in dict2"].append(d2[keys]) + + # compare meta + S.append("Different values in meta: \n") + different_meta = { + k: dict1["meta"][k] + for k in dict1["meta"] + if k in dict2["meta"] and dict1["meta"][k] != dict2["meta"][k] + } + + for keys in different_meta: + if keys not in conflicts: + conflicts[keys] = 1 + else: + conflicts[keys] = conflicts[keys] + 1 + + S.append(" Different values in %s \n" % str(keys)) + S.append(" dict1: %s \n" % str(dict1["meta"][str(keys)])) + S.append(" dict2: %s \n" % str(dict2["meta"][str(keys)])) + + # compare Measurements + S.append( + "Number of measurements in %s = %s \n" + % (dict1["meta"]["original_filename"], len(dict1["Measurements"])) + ) + S.append( + "Number of measurements in %s = %s \n" + % (dict2["meta"]["original_filename"], len(dict2["Measurements"])) + ) + S.append("Different values in Measurements:\n") + select_set = ["om", "Counts", "sigma"] + exlude_set = ["time", "Counts"] + + for keys1 in comp: + for key2 in dict1["Measurements"][str(comp[str(keys1)][0])]: + if key2 in exlude_set: + continue + if key2 not in select_set: + if ( + dict1["Measurements"][comp[str(keys1)][0]][str(key2)] + != dict2["Measurements"][str(comp[str(keys1)][0])][str(key2)] + ): + S.append( + "Measurement value " + "%s" + ", with hkl %s differs in meausrements %s and %s \n" + % (key2, keys1, comp[str(keys1)][0], comp[str(keys1)][1]) + ) + S.append( + " dict1: %s \n" + % str(dict1["Measurements"][comp[str(keys1)][0]][str(key2)]) + ) + S.append( + " dict2: %s \n" + % str(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) + ) + if key2 not in conflicts: + conflicts[key2] = {} + conflicts[key2]["amount"] = 1 + conflicts[key2]["measurements"] = str(comp[str(keys1)][0]) + else: + conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 + conflicts[key2]["measurements"] = ( + conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)][0])) + ) + else: + comparison = list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)]) == list( + dict2["Measurements"][comp[str(keys1)][1]][str(key2)] + ) + if len(list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)])) != len( + list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) + ): + if str("different length of %s" % key2) not in warnings: + warnings[str("different length of %s" % key2)] = list() + warnings[str("different length of %s" % key2)].append( + (str(comp[keys1][0]), str(comp[keys1][1])) + ) + else: + warnings[str("different length of %s" % key2)].append( + (str(comp[keys1][0]), str(comp[keys1][1])) + ) + if not comparison: + S.append( + "Measurement value " + "%s" + " differs in measurement %s and %s \n" + % (key2, comp[str(keys1)][0], comp[str(keys1)][1]) + ) + S.append( + " dict1: %s \n" + % str(list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)])) + ) + S.append( + " dict2: %s \n" + % str(list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)])) + ) + if key2 not in conflicts: + conflicts[key2] = {} + conflicts[key2]["amount"] = 1 + conflicts[key2]["measurements"] = str(comp[str(keys1)][0]) + else: + conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 + conflicts[key2]["measurements"] = ( + conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)][1])) + ) + + for keys in conflicts: + try: + conflicts[str(keys)]["measurements"] = conflicts[str(keys)]["measurements"].split(" ") + except: + continue + report_string = "".join(S) + return warnings, conflicts, report_string + + +def guess_next(dict1, dict2, comp): + """iterates thorough the scans and tries to decide if the scans should be substracted or merged""" + threshold = 0.05 + for keys in comp: + if ( + abs( + ( + dict1["Measurements"][str(comp[keys][0])]["temperature"] + - dict2["Measurements"][str(comp[keys][1])]["temperature"] + ) + / dict2["Measurements"][str(comp[keys][1])]["temperature"] + ) + < threshold + and abs( + ( + dict1["Measurements"][str(comp[keys][0])]["mag_field"] + - dict2["Measurements"][str(comp[keys][1])]["mag_field"] + ) + / dict2["Measurements"][str(comp[keys][1])]["mag_field"] + ) + < threshold + ): + comp[keys] = comp[keys] + tuple("m") + else: + comp[keys] = comp[keys] + tuple("s") + + return comp + + +def process_dict(dict1, dict2, comp): + """substracts or merges scans, guess_next function must run first """ + for keys in comp: + if comp[keys][2] == "s": + substract_measurement(dict1, dict2, comp[keys]) + elif comp[keys][2] == "m": + merge(dict1, dict2, comp[keys]) + + return dict1 From 1302e4f93fc0c6766b76dbdb7c814126f829ea31 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 25 Sep 2020 16:22:28 +0200 Subject: [PATCH 046/116] Delete ccl_dict_operation_v1.py --- ccl_dict_operation_v1.py | 416 --------------------------------------- 1 file changed, 416 deletions(-) delete mode 100644 ccl_dict_operation_v1.py diff --git a/ccl_dict_operation_v1.py b/ccl_dict_operation_v1.py deleted file mode 100644 index 1415321..0000000 --- a/ccl_dict_operation_v1.py +++ /dev/null @@ -1,416 +0,0 @@ -import numpy as np -import uncertainties as u -from fitvol2 import create_uncertanities - - -def scan_dict(dict): - """scans dictionary for duplicate hkl indexes - :arg dict : dictionary to scan - :return dictionary with matching scans, if there are none, the dict is empty - note: can be checked by "not d", true if empty - """ - d = {} - dupl = 0 - for i in dict["Measurements"]: - for j in dict["Measurements"]: - if dict["Measurements"][str(i)] != dict["Measurements"][str(j)]: - itup = ( - dict["Measurements"][str(i)]["h_index"], - dict["Measurements"][str(i)]["k_index"], - dict["Measurements"][str(i)]["l_index"], - ) - jtup = ( - dict["Measurements"][str(j)]["h_index"], - dict["Measurements"][str(j)]["k_index"], - dict["Measurements"][str(j)]["l_index"], - ) - if itup != jtup: - pass - else: - if str(itup) not in d: - d[str(itup)] = (i, j) - else: - dupl = dupl + 1 - d[str(itup) + "_" + str(dupl)] = (i, j) - else: - continue - return d - - -def compare_hkl(dict1, dict2): - """Compares two dictionaries based on hkl indexes and return dictionary with str(h k l) as key and tuple with - keys to same measurement in dict1 and dict2 - :arg dict1 : first dictionary - :arg dict2 : second dictionary - :return d : dict with matches - example of one key: '0.0 0.0 -1.0 : ('M1', 'M9')' meaning that 001 hkl measurement is M1 in first dict and M9 in second""" - d = {} - dupl = 0 - for keys in dict1["Measurements"]: - for key in dict2["Measurements"]: - if ( - dict1["Measurements"][str(keys)]["h_index"] - == dict2["Measurements"][str(key)]["h_index"] - and dict1["Measurements"][str(keys)]["k_index"] - == dict2["Measurements"][str(key)]["k_index"] - and dict1["Measurements"][str(keys)]["l_index"] - == dict2["Measurements"][str(key)]["l_index"] - ): - - if ( - str( - ( - str(dict1["Measurements"][str(keys)]["h_index"]) - + " " - + str(dict1["Measurements"][str(keys)]["k_index"]) - + " " - + str(dict1["Measurements"][str(keys)]["l_index"]) - ) - ) - not in d - ): - d[ - str( - str(dict1["Measurements"][str(keys)]["h_index"]) - + " " - + str(dict1["Measurements"][str(keys)]["k_index"]) - + " " - + str(dict1["Measurements"][str(keys)]["l_index"]) - ) - ] = (str(keys), str(key)) - else: - dupl = dupl + 1 - d[ - str( - str(dict1["Measurements"][str(keys)]["h_index"]) - + " " - + str(dict1["Measurements"][str(keys)]["k_index"]) - + " " - + str(dict1["Measurements"][str(keys)]["l_index"]) - + "_dupl" - + str(dupl) - ) - ] = (str(keys), str(key)) - else: - continue - - return d - - -def create_tuples(x, y, y_err): - """creates tuples for sorting and merginng of the data - Counts need to be normalized to monitor before""" - t = list() - for i in range(len(x)): - tup = (x[i], y[i], y_err) - t.append(tup) - return t - - -def merge(dict1, dict2, keys, auto=True, monitor=100000): - """merges the two tuples and sorts them, if om value is same, Counts value is average - averaging is propagated into sigma if dict1 == dict2, key[1] is deleted after merging - :arg dict1 : dictionary to which measurement will be merged - :arg dict2 : dictionary from which measurement will be merged - :arg keys : tuple with key to dict1 and dict2 - :arg auto : if true, when monitors are same, does not change it, if flase, takes monitor always - :arg monitor : final monitor after merging - note: dict1 and dict2 can be same dict - :return dict1 with merged scan""" - if auto: - if dict1["Measurements"][keys[0]]["monitor"] == dict2["Measurements"][keys[1]]["monitor"]: - monitor = dict1["Measurements"][keys[0]]["monitor"] - - # load om and Counts - x1, x2 = dict1["Measurements"][keys[0]]["om"], dict2["Measurements"][keys[1]]["om"] - y1, y2 = np.array(dict1["Measurements"][keys[0]]["Counts"]), np.array( - dict2["Measurements"][keys[1]]["Counts"] - ) - # normalize y to monitors - cor_y1 = (y1 / dict1["Measurements"][keys[0]]["monitor"]) * monitor - cor_y2 = (y2 / dict2["Measurements"][keys[0]]["monitor"]) * monitor - # check if sigma errors for y exist, otherwise create them as sqrt(y) - y_err1 = ( - np.sqrt(cor_y1) - if "sigma" not in dict1["Measurements"][keys[0]] - else np.array(dict1["Measurements"][keys[0]]["sigma"]) - * np.sqrt(monitor / dict1["Measurements"][keys[0]]["monitor"]) - ) - y_err2 = ( - np.sqrt(cor_y2) - if "sigma" not in dict2["Measurements"][keys[1]] - else np.array(dict2["Measurements"][keys[1]]["sigma"]) - * np.sqrt(monitor / dict2["Measurements"][keys[1]]["monitor"]) - ) - # creates touples (om, Counts, sigma) for sorting and further processing - tuple_list = create_tuples(x1, cor_y1, y_err1) + create_tuples(x2, cor_y2, y_err2) - # Sort the list on om and add 0 0 0 tuple to the last postion - sorted_t = sorted(tuple_list, key=lambda tup: tup[0]) - sorted_t.append((0, 0, 0)) - om, Counts, sigma = [], [], [] - seen = list() - for i in range(len(sorted_t) - 1): - if sorted_t[i][0] not in seen: - if sorted_t[i][0] != sorted_t[i + 1][0]: - om = np.append(om, sorted_t[i][0]) - Counts = np.append(Counts, sorted_t[i][1]) - sigma = np.append(sigma, sorted_t[i][2]) - else: - om = np.append(om, sorted_t[i][0]) - counts1, counts2 = sorted_t[i][1], sorted_t[i + 1][1] - sigma1, sigma2 = sorted_t[i][2], sorted_t[i + 1][2] - count_err1 = u.ufloat(counts1, sigma1) - count_err2 = u.ufloat(counts2, sigma2) - avg = (count_err1 + count_err2) / 2 - Counts = np.append(Counts, avg.n) - sigma = np.append(sigma, avg.s) - seen.append(sorted_t[i][0]) - else: - continue - - if dict1 == dict2: - del dict1["Measurements"][keys[1]] - - dict1["Measurements"][keys[0]]["om"] = om - dict1["Measurements"][keys[0]]["Counts"] = Counts - dict1["Measurements"][keys[0]]["sigma"] = sigma - return dict1 - - -def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): - """Substracts two measurement (measurement key2 from dict2 from measurent key1 in dict1), expects om to be same - :arg dict1 : dictionary to which measurement will be merged - :arg dict2 : dictionary from which measurement will be merged - :arg keys : tuple with key to dict1 and dict2 - :arg auto : if true, when monitors are same, does not change it, if flase, takes monitor always - :arg monitor : final monitor after merging - :returns d : dict1 with substracted Counts from dict2 and sigma that comes from the substraction""" - - if len(dict1["Measurements"][keys[0]]["om"]) != len(dict2["Measurements"][keys[1]]["om"]): - raise ValueError("Omegas have different lengths, cannot be substracted") - - if auto: - if dict1["Measurements"][keys[0]]["monitor"] == dict2["Measurements"][keys[1]]["monitor"]: - monitor = dict1["Measurements"][keys[0]]["monitor"] - - monitor_ratio_prim = monitor / dict1["Measurements"][str(keys[0])]["monitor"] - monitor_ratio_sec = monitor / dict2["Measurements"][str(keys[1])]["monitor"] - y1 = np.array(dict1["Measurements"][str(keys[0])]["Counts"]) * monitor_ratio_prim - y2 = np.array(dict2["Measurements"][str(keys[1])]["Counts"]) * monitor_ratio_sec - y_err1 = ( - np.sqrt(y1) - if "sigma" not in dict1["Measurements"][keys[0]] - else dict1["Measurements"][keys[0]]["sigma"] - * np.sqrt(monitor / dict1["Measurements"][keys[0]]["monitor"]) - ) - y_err2 = ( - np.sqrt(y2) - if "sigma" not in dict2["Measurements"][keys[1]] - else dict2["Measurements"][keys[1]]["sigma"] - * np.sqrt(monitor / dict2["Measurements"][keys[1]]["monitor"]) - ) - dict1_count_err = create_uncertanities(y1, y_err1) - dict2_count_err = create_uncertanities(y2, y_err2) - - res = np.subtract(dict1_count_err, dict2_count_err) - res_nom = [] - res_err = [] - for k in range(len(res)): - res_nom = np.append(res_nom, res[k].n) - res_err = np.append(res_err, res[k].s) - dict1["Measurements"][str(keys[0])]["Counts"] = res_nom - dict1["Measurements"][str(keys[0])]["sigma"] = res_err - - -def compare_dict(dict1, dict2): - """takes two ccl dictionaries and compare different values for each key - :arg dict1 : dictionary 1 (ccl) - :arg dict2 : dictionary 2 (ccl) - :returns warning : dictionary with keys from primary files (if they differ) with - information of how many measurement differ and which ones differ - :returns report_string string comparing all different values respecively of measurements""" - - if dict1["file_type"] != dict2["file_type"]: - print("select two dicts") - return - S = [] - conflicts = {} - warnings = {} - - comp = compare_hkl(dict1, dict2) - d1 = scan_dict(dict1) - d2 = scan_dict(dict2) - if not d1: - S.append("There are no duplicates in %s (dict1) \n" % dict1["meta"]["original_filename"]) - else: - S.append( - "There are %d duplicates in %s (dict1) \n" - % (len(d1), dict1["meta"]["original_filename"]) - ) - warnings["Duplicates in dict1"] = list() - for keys in d1: - S.append("Measurements %s with hkl %s \n" % (d1[keys], keys)) - warnings["Duplicates in dict1"].append(d1[keys]) - if not d2: - S.append("There are no duplicates in %s (dict2) \n" % dict2["meta"]["original_filename"]) - else: - S.append( - "There are %d duplicates in %s (dict2) \n" - % (len(d2), dict2["meta"]["original_filename"]) - ) - warnings["Duplicates in dict2"] = list() - for keys in d2: - S.append("Measurements %s with hkl %s \n" % (d2[keys], keys)) - warnings["Duplicates in dict2"].append(d2[keys]) - - # compare meta - S.append("Different values in meta: \n") - different_meta = { - k: dict1["meta"][k] - for k in dict1["meta"] - if k in dict2["meta"] and dict1["meta"][k] != dict2["meta"][k] - } - - for keys in different_meta: - if keys not in conflicts: - conflicts[keys] = 1 - else: - conflicts[keys] = conflicts[keys] + 1 - - S.append(" Different values in %s \n" % str(keys)) - S.append(" dict1: %s \n" % str(dict1["meta"][str(keys)])) - S.append(" dict2: %s \n" % str(dict2["meta"][str(keys)])) - - # compare Measurements - S.append( - "Number of measurements in %s = %s \n" - % (dict1["meta"]["original_filename"], len(dict1["Measurements"])) - ) - S.append( - "Number of measurements in %s = %s \n" - % (dict2["meta"]["original_filename"], len(dict2["Measurements"])) - ) - S.append("Different values in Measurements:\n") - select_set = ["om", "Counts", "sigma"] - exlude_set = ["time", "Counts"] - - for keys1 in comp: - for key2 in dict1["Measurements"][str(comp[str(keys1)][0])]: - if key2 in exlude_set: - continue - if key2 not in select_set: - if ( - dict1["Measurements"][comp[str(keys1)][0]][str(key2)] - != dict2["Measurements"][str(comp[str(keys1)][0])][str(key2)] - ): - S.append( - "Measurement value " - "%s" - ", with hkl %s differs in meausrements %s and %s \n" - % (key2, keys1, comp[str(keys1)][0], comp[str(keys1)][1]) - ) - S.append( - " dict1: %s \n" - % str(dict1["Measurements"][comp[str(keys1)][0]][str(key2)]) - ) - S.append( - " dict2: %s \n" - % str(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) - ) - if key2 not in conflicts: - conflicts[key2] = {} - conflicts[key2]["amount"] = 1 - conflicts[key2]["measurements"] = str(comp[str(keys1)][0]) - else: - conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 - conflicts[key2]["measurements"] = ( - conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)][0])) - ) - else: - comparison = list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)]) == list( - dict2["Measurements"][comp[str(keys1)][1]][str(key2)] - ) - if len(list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)])) != len( - list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) - ): - if str("different length of %s" % key2) not in warnings: - warnings[str("different length of %s" % key2)] = list() - warnings[str("different length of %s" % key2)].append( - (str(comp[keys1][0]), str(comp[keys1][1])) - ) - else: - warnings[str("different length of %s" % key2)].append( - (str(comp[keys1][0]), str(comp[keys1][1])) - ) - if not comparison: - S.append( - "Measurement value " - "%s" - " differs in measurement %s and %s \n" - % (key2, comp[str(keys1)][0], comp[str(keys1)][1]) - ) - S.append( - " dict1: %s \n" - % str(list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)])) - ) - S.append( - " dict2: %s \n" - % str(list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)])) - ) - if key2 not in conflicts: - conflicts[key2] = {} - conflicts[key2]["amount"] = 1 - conflicts[key2]["measurements"] = str(comp[str(keys1)][0]) - else: - conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 - conflicts[key2]["measurements"] = ( - conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)][1])) - ) - - for keys in conflicts: - try: - conflicts[str(keys)]["measurements"] = conflicts[str(keys)]["measurements"].split(" ") - except: - continue - report_string = "".join(S) - return warnings, conflicts, report_string - - -def guess_next(dict1, dict2, comp): - """iterates thorough the scans and tries to decide if the scans should be substracted or merged""" - threshold = 0.05 - for keys in comp: - if ( - abs( - ( - dict1["Measurements"][str(comp[keys][0])]["temperature"] - - dict2["Measurements"][str(comp[keys][1])]["temperature"] - ) - / dict2["Measurements"][str(comp[keys][1])]["temperature"] - ) - < threshold - and abs( - ( - dict1["Measurements"][str(comp[keys][0])]["mag_field"] - - dict2["Measurements"][str(comp[keys][1])]["mag_field"] - ) - / dict2["Measurements"][str(comp[keys][1])]["mag_field"] - ) - < threshold - ): - comp[keys] = comp[keys] + tuple("m") - else: - comp[keys] = comp[keys] + tuple("s") - - return comp - - -def process_dict(dict1, dict2, comp): - """substracts or merges scans, guess_next function must run first """ - for keys in comp: - if comp[keys][2] == "s": - substract_measurement(dict1, dict2, comp[keys]) - elif comp[keys][2] == "m": - merge(dict1, dict2, comp[keys]) - - return dict1 From 7afd41ea8bb67ba45374a26b7ffb6f7f278af828 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Tue, 29 Sep 2020 13:43:48 +0200 Subject: [PATCH 047/116] Update ccl_dict_operation.py Corrected some lines, no major changes --- pyzebra/ccl_dict_operation.py | 70 ++++++++++++++--------------------- 1 file changed, 28 insertions(+), 42 deletions(-) diff --git a/pyzebra/ccl_dict_operation.py b/pyzebra/ccl_dict_operation.py index ee79c89..db30c78 100644 --- a/pyzebra/ccl_dict_operation.py +++ b/pyzebra/ccl_dict_operation.py @@ -106,6 +106,21 @@ def create_tuples(x, y, y_err): t.append(tup) return t +def normalize(dict, key, monitor): + """Normalizes the measurement to monitor, checks if sigma exists, otherwise creates it + :arg dict : dictionary to from which to tkae the scan + :arg key : which scan to normalize from dict1 + :arg monitor : final monitor + :return counts - normalized counts + :return sigma - normalized sigma""" + + counts = np.array(dict["Measurements"][key]["Counts"]) + sigma = np.sqrt(counts) if "sigma" not in dict["Measurements"][key] else dict["Measurements"][key]['sigma'] + monitor_ratio = monitor/dict["Measurements"][key]["monitor"] + scaled_counts = counts*monitor_ratio + scaled_sigma = np.array(sigma)*monitor_ratio + + return scaled_counts, scaled_sigma def merge(dict1, dict2, keys, auto=True, monitor=100000): """merges the two tuples and sorts them, if om value is same, Counts value is average @@ -123,25 +138,8 @@ def merge(dict1, dict2, keys, auto=True, monitor=100000): # load om and Counts x1, x2 = dict1["Measurements"][keys[0]]["om"], dict2["Measurements"][keys[1]]["om"] - y1, y2 = np.array(dict1["Measurements"][keys[0]]["Counts"]), np.array( - dict2["Measurements"][keys[1]]["Counts"] - ) - # normalize y to monitors - cor_y1 = (y1 / dict1["Measurements"][keys[0]]["monitor"]) * monitor - cor_y2 = (y2 / dict2["Measurements"][keys[0]]["monitor"]) * monitor - # check if sigma errors for y exist, otherwise create them as sqrt(y) - y_err1 = ( - np.sqrt(cor_y1) - if "sigma" not in dict1["Measurements"][keys[0]] - else np.array(dict1["Measurements"][keys[0]]["sigma"]) - * np.sqrt(monitor / dict1["Measurements"][keys[0]]["monitor"]) - ) - y_err2 = ( - np.sqrt(cor_y2) - if "sigma" not in dict2["Measurements"][keys[1]] - else np.array(dict2["Measurements"][keys[1]]["sigma"]) - * np.sqrt(monitor / dict2["Measurements"][keys[1]]["monitor"]) - ) + cor_y1, y_err1 = normalize(dict1, keys[0], monitor=monitor) + cor_y2, y_err2 = normalize(dict2, keys[1], monitor=monitor) # creates touples (om, Counts, sigma) for sorting and further processing tuple_list = create_tuples(x1, cor_y1, y_err1) + create_tuples(x2, cor_y2, y_err2) # Sort the list on om and add 0 0 0 tuple to the last postion @@ -174,6 +172,7 @@ def merge(dict1, dict2, keys, auto=True, monitor=100000): dict1["Measurements"][keys[0]]["om"] = om dict1["Measurements"][keys[0]]["Counts"] = Counts dict1["Measurements"][keys[0]]["sigma"] = sigma + dict1["Measurements"][keys[0]]["monitor"] = monitor return dict1 @@ -193,28 +192,14 @@ def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): if dict1["Measurements"][keys[0]]["monitor"] == dict2["Measurements"][keys[1]]["monitor"]: monitor = dict1["Measurements"][keys[0]]["monitor"] - monitor_ratio_prim = monitor / dict1["Measurements"][str(keys[0])]["monitor"] - monitor_ratio_sec = monitor / dict2["Measurements"][str(keys[1])]["monitor"] - y1 = np.array(dict1["Measurements"][str(keys[0])]["Counts"]) * monitor_ratio_prim - y2 = np.array(dict2["Measurements"][str(keys[1])]["Counts"]) * monitor_ratio_sec - y_err1 = ( - np.sqrt(y1) - if "sigma" not in dict1["Measurements"][keys[0]] - else dict1["Measurements"][keys[0]]["sigma"] - * np.sqrt(monitor / dict1["Measurements"][keys[0]]["monitor"]) - ) - y_err2 = ( - np.sqrt(y2) - if "sigma" not in dict2["Measurements"][keys[1]] - else dict2["Measurements"][keys[1]]["sigma"] - * np.sqrt(monitor / dict2["Measurements"][keys[1]]["monitor"]) - ) - dict1_count_err = create_uncertanities(y1, y_err1) - dict2_count_err = create_uncertanities(y2, y_err2) - if np.average(y1) > np.average(y2): - res = np.subtract(dict1_count_err, dict2_count_err) - else: - res = np.subtract(dict2_count_err, dict1_count_err) + cor_y1, y_err1 = normalize(dict1, keys[0], monitor=monitor) + cor_y2, y_err2 = normalize(dict2, keys[1], monitor=monitor) + + dict1_count_err = create_uncertanities(cor_y1, y_err1) + dict2_count_err = create_uncertanities(cor_y2, y_err2) + + res = np.subtract(dict1_count_err, dict2_count_err) + res_nom = [] res_err = [] for k in range(len(res)): @@ -222,7 +207,8 @@ def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): res_err = np.append(res_err, res[k].s) dict1["Measurements"][str(keys[0])]["Counts"] = res_nom dict1["Measurements"][str(keys[0])]["sigma"] = res_err - + dict1["Measurements"][str(keys[0])]["monitor"] = monitor + return dict1 def compare_dict(dict1, dict2): """takes two ccl dictionaries and compare different values for each key From e3dd010c945cd921239dfdb402897f9e21d45bf2 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 29 Sep 2020 13:41:24 +0200 Subject: [PATCH 048/116] Add experimental code for saving data into a file --- pyzebra/app/panel_1D_detector.py | 37 ++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/pyzebra/app/panel_1D_detector.py b/pyzebra/app/panel_1D_detector.py index c18ed5f..d738e32 100644 --- a/pyzebra/app/panel_1D_detector.py +++ b/pyzebra/app/panel_1D_detector.py @@ -9,6 +9,7 @@ from bokeh.models import ( Button, Circle, ColumnDataSource, + CustomJS, DataRange1d, Div, FileInput, @@ -26,8 +27,25 @@ from bokeh.models import ( import pyzebra +javaScript = """ +setTimeout(function() { + const filename = 'output' + js_data.data['ext'] + const blob = new Blob([js_data.data['cont']], {type: 'text/plain'}) + const link = document.createElement('a'); + document.body.appendChild(link); + const url = window.URL.createObjectURL(blob); + link.href = url; + link.download = filename; + link.click(); + window.URL.revokeObjectURL(url); + document.body.removeChild(link); +}, 500); +""" + + def create(): det_data = {} + js_data = ColumnDataSource(data=dict(cont=[], ext=[])) def upload_button_callback(_attr, _old, new): nonlocal det_data @@ -119,13 +137,7 @@ def create(): process_button = Button(label="Process All", button_type="primary") process_button.on_click(process_button_callback) - output_textinput = TextAreaInput( - title="Content of comm/incomm file: (once fixed, Export button will save to user machine)", - width=600, - height=400, - ) - - def save_button_callback(): + def export_results(det_data): if det_data["meta"]["indices"] == "hkl": ext = ".comm" elif det_data["meta"]["indices"] == "real": @@ -136,15 +148,22 @@ def create(): pyzebra.export_comm(det_data, temp_file) with open(f"{temp_file}{ext}") as f: - output_textinput.value = f.read() + output_content = f.read() + + return output_content, ext + + def save_button_callback(): + cont, ext = export_results(det_data) + js_data.data.update(cont=[cont], ext=[ext]) save_button = Button(label="Export to .comm/.incomm file:") save_button.on_click(save_button_callback) + save_button.js_on_click(CustomJS(args={"js_data": js_data}, code=javaScript)) upload_div = Div(text="Upload .ccl file:") tab_layout = column( row(column(Spacer(height=5), upload_div), upload_button, meas_select), - row(plot, Spacer(width=30), fit_output_textinput, output_textinput), + row(plot, Spacer(width=30), fit_output_textinput), row(column(smooth_toggle, process_button, save_button)), ) From ddc5522d8b739b0ca4dd6e96a903f76311a5d2f1 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 29 Sep 2020 14:36:39 +0200 Subject: [PATCH 049/116] Display gauss and bkg fits --- pyzebra/app/panel_1D_detector.py | 13 +++++++++---- pyzebra/fit2.py | 2 ++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pyzebra/app/panel_1D_detector.py b/pyzebra/app/panel_1D_detector.py index d738e32..52db14e 100644 --- a/pyzebra/app/panel_1D_detector.py +++ b/pyzebra/app/panel_1D_detector.py @@ -70,15 +70,17 @@ def create(): num_of_peaks = meas.get("num_of_peaks") if num_of_peaks is not None and num_of_peaks > 0: plot_circle_source.data.update(x=meas["peak_indexes"], y=meas["peak_heights"]) - plot_smooth_source.data.update(x=x, y=meas["smooth_peaks"]) else: plot_circle_source.data.update(x=[], y=[]) - plot_smooth_source.data.update(x=[], y=[]) fit = meas.get("fit") if fit is not None: + plot_gauss_source.data.update(x=x, y=meas["fit"]["comps"]["gaussian"]) + plot_bkg_source.data.update(x=x, y=meas["fit"]["comps"]["background"]) fit_output_textinput.value = str(fit["full_report"]) else: + plot_gauss_source.data.update(x=[], y=[]) + plot_bkg_source.data.update(x=[], y=[]) fit_output_textinput.value = "" # Main plot @@ -99,8 +101,11 @@ def create(): plot_line_source = ColumnDataSource(dict(x=[0], y=[0])) plot.add_glyph(plot_line_source, Line(x="x", y="y", line_color="steelblue")) - plot_smooth_source = ColumnDataSource(dict(x=[0], y=[0])) - plot.add_glyph(plot_smooth_source, Line(x="x", y="y", line_color="red")) + plot_gauss_source = ColumnDataSource(dict(x=[0], y=[0])) + plot.add_glyph(plot_gauss_source, Line(x="x", y="y", line_color="red", line_dash="dashed")) + + plot_bkg_source = ColumnDataSource(dict(x=[0], y=[0])) + plot.add_glyph(plot_bkg_source, Line(x="x", y="y", line_color="green", line_dash="dashed")) plot_circle_source = ColumnDataSource(dict(x=[], y=[])) plot.add_glyph(plot_circle_source, Circle(x="x", y="y")) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index fd55659..1e414fc 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -220,6 +220,8 @@ def fitccl( d["int_area"] = int_area d["fit_area"] = fit_area d["full_report"] = result.fit_report() + d["result"] = result + d["comps"] = comps meas["fit"] = d return data From b5476fc9b02782d79963cea3cc8a77a5a90ca8e2 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 29 Sep 2020 15:13:20 +0200 Subject: [PATCH 050/116] Add input for proposal to simplify access data --- pyzebra/app/panel_1D_detector.py | 36 ++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/pyzebra/app/panel_1D_detector.py b/pyzebra/app/panel_1D_detector.py index 52db14e..2018bf8 100644 --- a/pyzebra/app/panel_1D_detector.py +++ b/pyzebra/app/panel_1D_detector.py @@ -21,6 +21,7 @@ from bokeh.models import ( Select, Spacer, TextAreaInput, + TextInput, Toggle, ) @@ -42,11 +43,39 @@ setTimeout(function() { }, 500); """ +PROPOSAL_PATH = "/afs/psi.ch/project/sinqdata/2020/zebra/" + def create(): det_data = {} js_data = ColumnDataSource(data=dict(cont=[], ext=[])) + def proposal_textinput_callback(_attr, _old, new): + ccl_path = os.path.join(PROPOSAL_PATH, new) + ccl_file_list = [] + for file in os.listdir(ccl_path): + if file.endswith(".ccl"): + ccl_file_list.append((os.path.join(ccl_path, file), file)) + ccl_file_select.options = ccl_file_list + ccl_file_select.value = ccl_file_list[0][0] + + proposal_textinput = TextInput(title="Enter proposal number:") + proposal_textinput.on_change("value", proposal_textinput_callback) + + def ccl_file_select_callback(_attr, _old, new): + nonlocal det_data + with open(new) as file: + _, ext = os.path.splitext(new) + det_data = pyzebra.parse_1D(file, ext) + + meas_list = list(det_data["Measurements"].keys()) + meas_select.options = meas_list + meas_select.value = None + meas_select.value = meas_list[0] + + ccl_file_select = Select(title="Available .ccl files") + ccl_file_select.on_change("value", ccl_file_select_callback) + def upload_button_callback(_attr, _old, new): nonlocal det_data with io.StringIO(base64.b64decode(new).decode()) as file: @@ -55,6 +84,7 @@ def create(): meas_list = list(det_data["Measurements"].keys()) meas_select.options = meas_list + meas_select.value = None meas_select.value = meas_list[0] upload_button = FileInput(accept=".ccl") @@ -112,7 +142,8 @@ def create(): # Measurement select def meas_select_callback(_attr, _old, new): - _update_plot(new) + if new is not None: + _update_plot(new) meas_select = Select() meas_select.on_change("value", meas_select_callback) @@ -165,8 +196,9 @@ def create(): save_button.on_click(save_button_callback) save_button.js_on_click(CustomJS(args={"js_data": js_data}, code=javaScript)) - upload_div = Div(text="Upload .ccl file:") + upload_div = Div(text="Or upload .ccl file:") tab_layout = column( + row(proposal_textinput, ccl_file_select), row(column(Spacer(height=5), upload_div), upload_button, meas_select), row(plot, Spacer(width=30), fit_output_textinput), row(column(smooth_toggle, process_button, save_button)), From b095b5762e5aca006b3b8b71b3395892634aaea8 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Wed, 30 Sep 2020 09:22:40 +0200 Subject: [PATCH 051/116] Update ccl_dict_operation.py added note to trace merging and sutraction --- pyzebra/ccl_dict_operation.py | 38 +++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/pyzebra/ccl_dict_operation.py b/pyzebra/ccl_dict_operation.py index db30c78..c185ea6 100644 --- a/pyzebra/ccl_dict_operation.py +++ b/pyzebra/ccl_dict_operation.py @@ -38,12 +38,13 @@ def scan_dict(dict): def compare_hkl(dict1, dict2): - """Compares two dictionaries based on hkl indexes and return dictionary with str(h k l) as key and tuple with - keys to same measurement in dict1 and dict2 + """Compares two dictionaries based on hkl indexes and return dictionary with str(h k l) as + key and tuple with keys to same measurement in dict1 and dict2 :arg dict1 : first dictionary :arg dict2 : second dictionary :return d : dict with matches - example of one key: '0.0 0.0 -1.0 : ('M1', 'M9')' meaning that 001 hkl measurement is M1 in first dict and M9 in second""" + example of one key: '0.0 0.0 -1.0 : ('M1', 'M9')' meaning that 001 hkl measurement is M1 in + first dict and M9 in second""" d = {} dupl = 0 for keys in dict1["Measurements"]: @@ -106,6 +107,7 @@ def create_tuples(x, y, y_err): t.append(tup) return t + def normalize(dict, key, monitor): """Normalizes the measurement to monitor, checks if sigma exists, otherwise creates it :arg dict : dictionary to from which to tkae the scan @@ -115,13 +117,18 @@ def normalize(dict, key, monitor): :return sigma - normalized sigma""" counts = np.array(dict["Measurements"][key]["Counts"]) - sigma = np.sqrt(counts) if "sigma" not in dict["Measurements"][key] else dict["Measurements"][key]['sigma'] - monitor_ratio = monitor/dict["Measurements"][key]["monitor"] - scaled_counts = counts*monitor_ratio - scaled_sigma = np.array(sigma)*monitor_ratio + sigma = ( + np.sqrt(counts) + if "sigma" not in dict["Measurements"][key] + else dict["Measurements"][key]["sigma"] + ) + monitor_ratio = monitor / dict["Measurements"][key]["monitor"] + scaled_counts = counts * monitor_ratio + scaled_sigma = np.array(sigma) * monitor_ratio return scaled_counts, scaled_sigma + def merge(dict1, dict2, keys, auto=True, monitor=100000): """merges the two tuples and sorts them, if om value is same, Counts value is average averaging is propagated into sigma if dict1 == dict2, key[1] is deleted after merging @@ -169,6 +176,13 @@ def merge(dict1, dict2, keys, auto=True, monitor=100000): if dict1 == dict2: del dict1["Measurements"][keys[1]] + note = f'This measurement was merged with measurement {keys[1]} from ' \ + f'file {dict2["meta"]["original_filename"]} \n' + if "notes" not in dict1["Measurements"][str(keys[0])]: + dict1["Measurements"][str(keys[0])]["notes"] = note + else: + dict1["Measurements"][str(keys[0])]["notes"] += note + dict1["Measurements"][keys[0]]["om"] = om dict1["Measurements"][keys[0]]["Counts"] = Counts dict1["Measurements"][keys[0]]["sigma"] = sigma @@ -208,8 +222,15 @@ def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): dict1["Measurements"][str(keys[0])]["Counts"] = res_nom dict1["Measurements"][str(keys[0])]["sigma"] = res_err dict1["Measurements"][str(keys[0])]["monitor"] = monitor + note = f'Measurement {keys[1]} from file {dict2["meta"]["original_filename"]} ' \ + f'was substracted from this measurement \n' + if "notes" not in dict1["Measurements"][str(keys[0])]: + dict1["Measurements"][str(keys[0])]["notes"] = note + else: + dict1["Measurements"][str(keys[0])]["notes"] += note return dict1 + def compare_dict(dict1, dict2): """takes two ccl dictionaries and compare different values for each key :arg dict1 : dictionary 1 (ccl) @@ -365,7 +386,8 @@ def compare_dict(dict1, dict2): def guess_next(dict1, dict2, comp): - """iterates thorough the scans and tries to decide if the scans should be substracted or merged""" + """iterates thorough the scans and tries to decide if the scans should be + substracted or merged""" threshold = 0.05 for keys in comp: if ( From d7ce156db58077cb3e6018cdb7ca5e7fbaecce95 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Wed, 30 Sep 2020 10:06:16 +0200 Subject: [PATCH 052/116] Update ccl_dict_operation.py Added warning for negative numbers after subtraction --- pyzebra/ccl_dict_operation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyzebra/ccl_dict_operation.py b/pyzebra/ccl_dict_operation.py index c185ea6..6855e14 100644 --- a/pyzebra/ccl_dict_operation.py +++ b/pyzebra/ccl_dict_operation.py @@ -219,6 +219,10 @@ def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): for k in range(len(res)): res_nom = np.append(res_nom, res[k].n) res_err = np.append(res_err, res[k].s) + + if len([num for num in res_nom if num < 0]) >= 0.3*len(res_nom): + print(f'Warning! percentage of negative numbers in measurement subsracted {keys[0]} is ' + f'{len([num for num in res_nom if num < 0])/len(res_nom)}') dict1["Measurements"][str(keys[0])]["Counts"] = res_nom dict1["Measurements"][str(keys[0])]["sigma"] = res_err dict1["Measurements"][str(keys[0])]["monitor"] = monitor From dc7c2afdc8c687e16627db4806c0b50d4da733c6 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Mon, 5 Oct 2020 15:24:11 +0200 Subject: [PATCH 053/116] Update comm_export.py fixed some bugs --- pyzebra/comm_export.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py index 73d100d..8211dac 100644 --- a/pyzebra/comm_export.py +++ b/pyzebra/comm_export.py @@ -34,15 +34,14 @@ def export_comm(data, path, lorentz=False): if "fit" not in meas: print("Measurement skipped - no fit value for:", keys) continue - meas_number_str = f"{keys[1:]:{align}{padding[0]}}" h_str = f'{int(meas["h_index"]):{padding[1]}}' k_str = f'{int(meas["k_index"]):{padding[1]}}' l_str = f'{int(meas["l_index"]):{padding[1]}}' if meas["fit"]["export_fit"] is True: - area = float(meas["fit"]["g_amp"].value) + area = float(meas["fit"]["fit_area"].n) sigma_str = ( - f'{"{:8.2f}".format(float(meas["fit"]["g_amp"].stderr)):{align}{padding[2]}}' + f'{"{:8.2f}".format(float(meas["fit"]["fit_area"].s)):{align}{padding[2]}}' ) else: area = float(meas["fit"]["int_area"].n) From e0227fb7e6dd8bf0d0f76aade9f2021b9111d4ed Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Mon, 5 Oct 2020 15:27:13 +0200 Subject: [PATCH 054/116] Update fit2.py --- pyzebra/fit2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 1e414fc..8468c62 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -218,7 +218,7 @@ def fitccl( # ["export_fit"] = False if user wants num. int. value in comm/incomm, otherwise true d["ratio"] = (result.params["g_amp"].value - int_area.n) / result.params["g_amp"].value d["int_area"] = int_area - d["fit_area"] = fit_area + d["fit_area"] = u.ufloat(result.params["g_amp"].value, result.params["g_amp"].stderr) d["full_report"] = result.fit_report() d["result"] = result d["comps"] = comps From 456a15a365d2b79bd73f96cf94d43c705c829f50 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Mon, 5 Oct 2020 15:32:59 +0200 Subject: [PATCH 055/116] Update load_1D.py added the data type to dict, it is necessary for adding dat files into ccl dicts --- pyzebra/load_1D.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 0d049a5..2d83a41 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -18,8 +18,10 @@ META_VARS_STR = ( "proposal_user", "proposal_title", "proposal_email", + "detectorDistance", ) META_VARS_FLOAT = ( + "omega", "mf", "2-theta", "chi", @@ -40,7 +42,6 @@ META_VARS_FLOAT = ( "mcvl", "momu", "mcvu", - "detectorDistance", "snv", "snh", "snvm", @@ -96,6 +97,7 @@ def parse_1D(fileobj, data_type): # read data if data_type == ".ccl": + metadata['data_type'] = data_type measurements = {} decimal = list() data = fileobj.readlines() @@ -160,6 +162,7 @@ def parse_1D(fileobj, data_type): elif data_type == ".dat": # skip the first 2 rows, the third row contans the column names + metadata['data_type'] = data_type next(fileobj) next(fileobj) col_names = next(fileobj).split() From fdcaebb250e4084f454205384c6c258c6f3c36d3 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Mon, 5 Oct 2020 12:13:51 +0200 Subject: [PATCH 056/116] Rename panels --- pyzebra/app/app.py | 14 +++++++------- ...panel_1D_detector.py => panel_ccl_integrate.py} | 2 +- .../app/{panel_anatric.py => panel_hdf_anatric.py} | 2 +- .../{panel_data_viewer.py => panel_hdf_viewer.py} | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) rename pyzebra/app/{panel_1D_detector.py => panel_ccl_integrate.py} (99%) rename pyzebra/app/{panel_anatric.py => panel_hdf_anatric.py} (99%) rename pyzebra/app/{panel_data_viewer.py => panel_hdf_viewer.py} (99%) diff --git a/pyzebra/app/app.py b/pyzebra/app/app.py index 7beb17a..67f4e14 100644 --- a/pyzebra/app/app.py +++ b/pyzebra/app/app.py @@ -3,9 +3,9 @@ import argparse from bokeh.io import curdoc from bokeh.models import Tabs -import panel_anatric -import panel_data_viewer -import panel_1D_detector +import panel_hdf_anatric +import panel_hdf_viewer +import panel_ccl_integrate parser = argparse.ArgumentParser( prog="pyzebra", formatter_class=argparse.ArgumentDefaultsHelpFormatter @@ -17,8 +17,8 @@ doc = curdoc() doc.title = "pyzebra" # Final layout -tab_data_viewer = panel_data_viewer.create() -tab_anatric = panel_anatric.create() -tab_1D_detector = panel_1D_detector.create() +tab_hdf_viewer = panel_hdf_viewer.create() +tab_hdf_anatric = panel_hdf_anatric.create() +tab_ccl_integrate = panel_ccl_integrate.create() -doc.add_root(Tabs(tabs=[tab_data_viewer, tab_anatric, tab_1D_detector])) +doc.add_root(Tabs(tabs=[tab_hdf_viewer, tab_hdf_anatric, tab_ccl_integrate])) diff --git a/pyzebra/app/panel_1D_detector.py b/pyzebra/app/panel_ccl_integrate.py similarity index 99% rename from pyzebra/app/panel_1D_detector.py rename to pyzebra/app/panel_ccl_integrate.py index 2018bf8..6f23949 100644 --- a/pyzebra/app/panel_1D_detector.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -204,4 +204,4 @@ def create(): row(column(smooth_toggle, process_button, save_button)), ) - return Panel(child=tab_layout, title="1D Detector") + return Panel(child=tab_layout, title="ccl integrate") diff --git a/pyzebra/app/panel_anatric.py b/pyzebra/app/panel_hdf_anatric.py similarity index 99% rename from pyzebra/app/panel_anatric.py rename to pyzebra/app/panel_hdf_anatric.py index 77a019c..5023161 100644 --- a/pyzebra/app/panel_anatric.py +++ b/pyzebra/app/panel_hdf_anatric.py @@ -406,4 +406,4 @@ def create(): curdoc().add_periodic_callback(update_config, 1000) - return Panel(child=tab_layout, title="Anatric") + return Panel(child=tab_layout, title="hdf anatric") diff --git a/pyzebra/app/panel_data_viewer.py b/pyzebra/app/panel_hdf_viewer.py similarity index 99% rename from pyzebra/app/panel_data_viewer.py rename to pyzebra/app/panel_hdf_viewer.py index 9439474..9c3e87e 100644 --- a/pyzebra/app/panel_data_viewer.py +++ b/pyzebra/app/panel_hdf_viewer.py @@ -464,7 +464,7 @@ def create(): column(roi_avg_plot, layout_image, row(colormap_layout, hkl_layout)), ) - return Panel(child=tab_layout, title="Data Viewer") + return Panel(child=tab_layout, title="hdf viewer") def calculate_hkl(det_data, index, setup_type="nb_bi"): From 15428662f7119c2620bb68e637d97fc9d0dbb003 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Mon, 5 Oct 2020 14:05:27 +0200 Subject: [PATCH 057/116] Replace measurement Select with DataTable --- pyzebra/app/panel_ccl_integrate.py | 38 +++++++++++++++++++----------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 6f23949..2907aed 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -11,6 +11,7 @@ from bokeh.models import ( ColumnDataSource, CustomJS, DataRange1d, + DataTable, Div, FileInput, Grid, @@ -20,6 +21,7 @@ from bokeh.models import ( Plot, Select, Spacer, + TableColumn, TextAreaInput, TextInput, Toggle, @@ -69,9 +71,9 @@ def create(): det_data = pyzebra.parse_1D(file, ext) meas_list = list(det_data["Measurements"].keys()) - meas_select.options = meas_list - meas_select.value = None - meas_select.value = meas_list[0] + meas_table_source.data.update(measurement=meas_list) + meas_table_source.selected.indices = [] + meas_table_source.selected.indices = [0] ccl_file_select = Select(title="Available .ccl files") ccl_file_select.on_change("value", ccl_file_select_callback) @@ -83,9 +85,9 @@ def create(): det_data = pyzebra.parse_1D(file, ext) meas_list = list(det_data["Measurements"].keys()) - meas_select.options = meas_list - meas_select.value = None - meas_select.value = meas_list[0] + meas_table_source.data.update(measurement=meas_list) + meas_table_source.selected.indices = [] + meas_table_source.selected.indices = [0] upload_button = FileInput(accept=".ccl") upload_button.on_change("value", upload_button_callback) @@ -141,12 +143,19 @@ def create(): plot.add_glyph(plot_circle_source, Circle(x="x", y="y")) # Measurement select - def meas_select_callback(_attr, _old, new): - if new is not None: - _update_plot(new) + def meas_table_callback(_attr, _old, new): + if new: + _update_plot(meas_table_source.data["measurement"][new[-1]]) - meas_select = Select() - meas_select.on_change("value", meas_select_callback) + meas_table_source = ColumnDataSource(dict(measurement=[])) + meas_table = DataTable( + source=meas_table_source, + columns=[TableColumn(field="measurement", title="Measurement")], + width=100, + index_position=None, + ) + + meas_table_source.selected.on_change("indices", meas_table_callback) smooth_toggle = Toggle(label="Smooth curve") @@ -168,7 +177,8 @@ def create(): constraints_max=[None, None, None, None, None], ) - _update_plot(meas_select.value) + sel_ind = meas_table_source.selected.indices[-1] + _update_plot(meas_table_source.data["measurement"][sel_ind]) process_button = Button(label="Process All", button_type="primary") process_button.on_click(process_button_callback) @@ -199,8 +209,8 @@ def create(): upload_div = Div(text="Or upload .ccl file:") tab_layout = column( row(proposal_textinput, ccl_file_select), - row(column(Spacer(height=5), upload_div), upload_button, meas_select), - row(plot, Spacer(width=30), fit_output_textinput), + row(column(Spacer(height=5), upload_div), upload_button), + row(meas_table, plot, Spacer(width=30), fit_output_textinput), row(column(smooth_toggle, process_button, save_button)), ) From 4091d4ba320860f17fe79362c846837b54136b45 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Mon, 5 Oct 2020 14:33:05 +0200 Subject: [PATCH 058/116] Add find peak controls --- pyzebra/app/panel_ccl_integrate.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 2907aed..131481b 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -21,6 +21,7 @@ from bokeh.models import ( Plot, Select, Spacer, + Spinner, TableColumn, TextAreaInput, TextInput, @@ -157,14 +158,28 @@ def create(): meas_table_source.selected.on_change("indices", meas_table_callback) + peak_int_ratio_spinner = Spinner( + title="Peak intensity ratio:", value=0.8, step=0.01, low=0, high=1, default_size=145 + ) + peak_prominence_spinner = Spinner(title="Peak prominence:", value=50, low=0, default_size=145) smooth_toggle = Toggle(label="Smooth curve") + window_size_spinner = Spinner(title="Window size:", value=7, step=2, low=1, default_size=145) + poly_order_spinner = Spinner(title="Poly order:", value=3, low=0, default_size=145) fit_output_textinput = TextAreaInput(title="Fit results:", width=600, height=400) def process_button_callback(): nonlocal det_data for meas in det_data["Measurements"]: - det_data = pyzebra.ccl_findpeaks(det_data, meas, smooth=smooth_toggle.active) + det_data = pyzebra.ccl_findpeaks( + det_data, + meas, + int_threshold=peak_int_ratio_spinner.value, + prominence=peak_prominence_spinner.value, + smooth=smooth_toggle.active, + window_size=window_size_spinner.value, + poly_order=poly_order_spinner.value, + ) num_of_peaks = det_data["Measurements"][meas].get("num_of_peaks") if num_of_peaks is not None and num_of_peaks == 1: @@ -206,12 +221,18 @@ def create(): save_button.on_click(save_button_callback) save_button.js_on_click(CustomJS(args={"js_data": js_data}, code=javaScript)) + findpeak_controls = column( + row(peak_int_ratio_spinner, peak_prominence_spinner), + smooth_toggle, + row(window_size_spinner, poly_order_spinner), + ) + upload_div = Div(text="Or upload .ccl file:") tab_layout = column( row(proposal_textinput, ccl_file_select), row(column(Spacer(height=5), upload_div), upload_button), row(meas_table, plot, Spacer(width=30), fit_output_textinput), - row(column(smooth_toggle, process_button, save_button)), + row(column(findpeak_controls, process_button, save_button)), ) return Panel(child=tab_layout, title="ccl integrate") From f1e9b7d889a21bf96ab591dbb8f5f06a246bcd8b Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Mon, 5 Oct 2020 14:46:22 +0200 Subject: [PATCH 059/116] Split process into peak find and fit steps --- pyzebra/app/panel_ccl_integrate.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 131481b..40c5223 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -168,7 +168,7 @@ def create(): fit_output_textinput = TextAreaInput(title="Fit results:", width=600, height=400) - def process_button_callback(): + def peakfind_all_button_callback(): nonlocal det_data for meas in det_data["Measurements"]: det_data = pyzebra.ccl_findpeaks( @@ -181,6 +181,15 @@ def create(): poly_order=poly_order_spinner.value, ) + sel_ind = meas_table_source.selected.indices[-1] + _update_plot(meas_table_source.data["measurement"][sel_ind]) + + peakfind_all_button = Button(label="Peak Find All", button_type="primary") + peakfind_all_button.on_click(peakfind_all_button_callback) + + def fit_all_button_callback(): + nonlocal det_data + for meas in det_data["Measurements"]: num_of_peaks = det_data["Measurements"][meas].get("num_of_peaks") if num_of_peaks is not None and num_of_peaks == 1: det_data = pyzebra.fitccl( @@ -195,8 +204,8 @@ def create(): sel_ind = meas_table_source.selected.indices[-1] _update_plot(meas_table_source.data["measurement"][sel_ind]) - process_button = Button(label="Process All", button_type="primary") - process_button.on_click(process_button_callback) + fit_all_button = Button(label="Fit All", button_type="primary") + fit_all_button.on_click(fit_all_button_callback) def export_results(det_data): if det_data["meta"]["indices"] == "hkl": @@ -225,6 +234,7 @@ def create(): row(peak_int_ratio_spinner, peak_prominence_spinner), smooth_toggle, row(window_size_spinner, poly_order_spinner), + peakfind_all_button, ) upload_div = Div(text="Or upload .ccl file:") @@ -232,7 +242,7 @@ def create(): row(proposal_textinput, ccl_file_select), row(column(Spacer(height=5), upload_div), upload_button), row(meas_table, plot, Spacer(width=30), fit_output_textinput), - row(column(findpeak_controls, process_button, save_button)), + row(findpeak_controls, column(fit_all_button), column(save_button)), ) return Panel(child=tab_layout, title="ccl integrate") From 7df408eff182e6cd268dc54e4a98f1ba7482db2b Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Mon, 5 Oct 2020 15:15:08 +0200 Subject: [PATCH 060/116] Display number of peaks for measurements --- pyzebra/app/panel_ccl_integrate.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 40c5223..1f43a7e 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -72,7 +72,7 @@ def create(): det_data = pyzebra.parse_1D(file, ext) meas_list = list(det_data["Measurements"].keys()) - meas_table_source.data.update(measurement=meas_list) + meas_table_source.data.update(measurement=meas_list, peaks=[0] * len(meas_list)) meas_table_source.selected.indices = [] meas_table_source.selected.indices = [0] @@ -86,7 +86,7 @@ def create(): det_data = pyzebra.parse_1D(file, ext) meas_list = list(det_data["Measurements"].keys()) - meas_table_source.data.update(measurement=meas_list) + meas_table_source.data.update(measurement=meas_list, peaks=[0] * len(meas_list)) meas_table_source.selected.indices = [] meas_table_source.selected.indices = [0] @@ -148,10 +148,13 @@ def create(): if new: _update_plot(meas_table_source.data["measurement"][new[-1]]) - meas_table_source = ColumnDataSource(dict(measurement=[])) + meas_table_source = ColumnDataSource(dict(measurement=[], peaks=[])) meas_table = DataTable( source=meas_table_source, - columns=[TableColumn(field="measurement", title="Measurement")], + columns=[ + TableColumn(field="measurement", title="Meas"), + TableColumn(field="peaks", title="Peaks"), + ], width=100, index_position=None, ) @@ -181,6 +184,9 @@ def create(): poly_order=poly_order_spinner.value, ) + num_of_peaks = [meas["num_of_peaks"] for meas in det_data["Measurements"].values()] + meas_table_source.data.update(peaks=num_of_peaks) + sel_ind = meas_table_source.selected.indices[-1] _update_plot(meas_table_source.data["measurement"][sel_ind]) From 8b491aa300815fc6c41aceda2a5113e0b1a5d8ed Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Mon, 5 Oct 2020 16:43:20 +0200 Subject: [PATCH 061/116] Add possibility to change peak position --- pyzebra/app/panel_ccl_integrate.py | 38 +++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 1f43a7e..4949202 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -51,6 +51,7 @@ PROPOSAL_PATH = "/afs/psi.ch/project/sinqdata/2020/zebra/" def create(): det_data = {} + peak_pos_textinput_lock = False js_data = ColumnDataSource(data=dict(cont=[], ext=[])) def proposal_textinput_callback(_attr, _old, new): @@ -93,7 +94,14 @@ def create(): upload_button = FileInput(accept=".ccl") upload_button.on_change("value", upload_button_callback) + def _update_table(): + num_of_peaks = [meas["num_of_peaks"] for meas in det_data["Measurements"].values()] + meas_table_source.data.update(peaks=num_of_peaks) + def _update_plot(ind): + nonlocal peak_pos_textinput_lock + peak_pos_textinput_lock = True + meas = det_data["Measurements"][ind] y = meas["Counts"] x = list(range(len(y))) @@ -102,10 +110,19 @@ def create(): num_of_peaks = meas.get("num_of_peaks") if num_of_peaks is not None and num_of_peaks > 0: - plot_circle_source.data.update(x=meas["peak_indexes"], y=meas["peak_heights"]) + peak_indexes = meas["peak_indexes"] + if len(peak_indexes) == 1: + peak_pos_textinput.value = str(peak_indexes[0]) + else: + peak_pos_textinput.value = str(peak_indexes) + + plot_circle_source.data.update(x=peak_indexes, y=meas["peak_heights"]) else: + peak_pos_textinput.value = None plot_circle_source.data.update(x=[], y=[]) + peak_pos_textinput_lock = False + fit = meas.get("fit") if fit is not None: plot_gauss_source.data.update(x=x, y=meas["fit"]["comps"]["gaussian"]) @@ -161,6 +178,21 @@ def create(): meas_table_source.selected.on_change("indices", meas_table_callback) + def peak_pos_textinput_callback(_attr, _old, new): + if new is not None and not peak_pos_textinput_lock: + sel_ind = meas_table_source.selected.indices[-1] + meas_name = meas_table_source.data["measurement"][sel_ind] + meas = det_data["Measurements"][meas_name] + + meas["num_of_peaks"] = 1 + meas["peak_indexes"] = [float(new)] + meas["peak_heights"] = [0] + _update_table() + _update_plot(meas_name) + + peak_pos_textinput = TextInput(title="Peak position:") + peak_pos_textinput.on_change("value", peak_pos_textinput_callback) + peak_int_ratio_spinner = Spinner( title="Peak intensity ratio:", value=0.8, step=0.01, low=0, high=1, default_size=145 ) @@ -184,8 +216,7 @@ def create(): poly_order=poly_order_spinner.value, ) - num_of_peaks = [meas["num_of_peaks"] for meas in det_data["Measurements"].values()] - meas_table_source.data.update(peaks=num_of_peaks) + _update_table() sel_ind = meas_table_source.selected.indices[-1] _update_plot(meas_table_source.data["measurement"][sel_ind]) @@ -237,6 +268,7 @@ def create(): save_button.js_on_click(CustomJS(args={"js_data": js_data}, code=javaScript)) findpeak_controls = column( + peak_pos_textinput, row(peak_int_ratio_spinner, peak_prominence_spinner), smooth_toggle, row(window_size_spinner, poly_order_spinner), From 29a5126377313a8fca0318e41328751b214b7139 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Mon, 5 Oct 2020 17:23:26 +0200 Subject: [PATCH 062/116] Add peak fit controls --- pyzebra/app/panel_ccl_integrate.py | 85 ++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 5 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 4949202..c4fba7f 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -201,6 +201,27 @@ def create(): window_size_spinner = Spinner(title="Window size:", value=7, step=2, low=1, default_size=145) poly_order_spinner = Spinner(title="Poly order:", value=3, low=0, default_size=145) + centre_guess = Spinner(default_size=100) + centre_vary = Toggle(default_size=100, active=True) + centre_min = Spinner(default_size=100) + centre_max = Spinner(default_size=100) + sigma_guess = Spinner(default_size=100) + sigma_vary = Toggle(default_size=100, active=True) + sigma_min = Spinner(default_size=100) + sigma_max = Spinner(default_size=100) + ampl_guess = Spinner(default_size=100) + ampl_vary = Toggle(default_size=100, active=True) + ampl_min = Spinner(default_size=100) + ampl_max = Spinner(default_size=100) + slope_guess = Spinner(default_size=100) + slope_vary = Toggle(default_size=100, active=True) + slope_min = Spinner(default_size=100) + slope_max = Spinner(default_size=100) + offset_guess = Spinner(default_size=100) + offset_vary = Toggle(default_size=100, active=True) + offset_min = Spinner(default_size=100) + offset_max = Spinner(default_size=100) + fit_output_textinput = TextAreaInput(title="Fit results:", width=600, height=400) def peakfind_all_button_callback(): @@ -232,10 +253,34 @@ def create(): det_data = pyzebra.fitccl( det_data, meas, - guess=[None, None, None, None, None], - vary=[True, True, True, True, True], - constraints_min=[None, None, None, None, None], - constraints_max=[None, None, None, None, None], + guess=[ + centre_guess.value, + sigma_guess.value, + ampl_guess.value, + slope_guess.value, + offset_guess.value, + ], + vary=[ + centre_vary.active, + sigma_vary.active, + ampl_vary.active, + slope_vary.active, + offset_vary.active, + ], + constraints_min=[ + centre_min.value, + sigma_min.value, + ampl_min.value, + slope_min.value, + offset_min.value, + ], + constraints_max=[ + centre_max.value, + sigma_max.value, + ampl_max.value, + slope_max.value, + offset_max.value, + ], ) sel_ind = meas_table_source.selected.indices[-1] @@ -275,12 +320,42 @@ def create(): peakfind_all_button, ) + div_1 = Div(text="Guess:") + div_2 = Div(text="Vary:") + div_3 = Div(text="Min:") + div_4 = Div(text="Max:") + div_5 = Div(text="Gauss Centre:") + div_6 = Div(text="Gauss Sigma:") + div_7 = Div(text="Gauss Ampl.:") + div_8 = Div(text="Slope:") + div_9 = Div(text="Offset:") + fitpeak_controls = column( + row( + column( + Spacer(height=36), + div_1, + Spacer(height=12), + div_2, + Spacer(height=12), + div_3, + Spacer(height=12), + div_4, + ), + column(div_5, centre_guess, centre_vary, centre_min, centre_max), + column(div_6, sigma_guess, sigma_vary, sigma_min, sigma_max), + column(div_7, ampl_guess, ampl_vary, ampl_min, ampl_max), + column(div_8, slope_guess, slope_vary, slope_min, slope_max), + column(div_9, offset_guess, offset_vary, offset_min, offset_max), + ), + row(fit_all_button), + ) + upload_div = Div(text="Or upload .ccl file:") tab_layout = column( row(proposal_textinput, ccl_file_select), row(column(Spacer(height=5), upload_div), upload_button), row(meas_table, plot, Spacer(width=30), fit_output_textinput), - row(findpeak_controls, column(fit_all_button), column(save_button)), + row(findpeak_controls, Spacer(width=30), fitpeak_controls, column(save_button)), ) return Panel(child=tab_layout, title="ccl integrate") From 62ecc54ee49af5ef724349e114596e39c013a70a Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 6 Oct 2020 11:15:37 +0200 Subject: [PATCH 063/116] Add find peak and fit for currently selected data --- pyzebra/app/panel_ccl_integrate.py | 69 +++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index c4fba7f..d3d3c52 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -95,7 +95,7 @@ def create(): upload_button.on_change("value", upload_button_callback) def _update_table(): - num_of_peaks = [meas["num_of_peaks"] for meas in det_data["Measurements"].values()] + num_of_peaks = [meas.get("num_of_peaks", 0) for meas in det_data["Measurements"].values()] meas_table_source.data.update(peaks=num_of_peaks) def _update_plot(ind): @@ -245,6 +245,26 @@ def create(): peakfind_all_button = Button(label="Peak Find All", button_type="primary") peakfind_all_button.on_click(peakfind_all_button_callback) + def peakfind_button_callback(): + nonlocal det_data + sel_ind = meas_table_source.selected.indices[-1] + meas = meas_table_source.data["measurement"][sel_ind] + det_data = pyzebra.ccl_findpeaks( + det_data, + meas, + int_threshold=peak_int_ratio_spinner.value, + prominence=peak_prominence_spinner.value, + smooth=smooth_toggle.active, + window_size=window_size_spinner.value, + poly_order=poly_order_spinner.value, + ) + + _update_table() + _update_plot(meas) + + peakfind_button = Button(label="Peak Find Current") + peakfind_button.on_click(peakfind_button_callback) + def fit_all_button_callback(): nonlocal det_data for meas in det_data["Measurements"]: @@ -289,6 +309,51 @@ def create(): fit_all_button = Button(label="Fit All", button_type="primary") fit_all_button.on_click(fit_all_button_callback) + def fit_button_callback(): + nonlocal det_data + sel_ind = meas_table_source.selected.indices[-1] + meas = meas_table_source.data["measurement"][sel_ind] + + num_of_peaks = det_data["Measurements"][meas].get("num_of_peaks") + if num_of_peaks is not None and num_of_peaks == 1: + det_data = pyzebra.fitccl( + det_data, + meas, + guess=[ + centre_guess.value, + sigma_guess.value, + ampl_guess.value, + slope_guess.value, + offset_guess.value, + ], + vary=[ + centre_vary.active, + sigma_vary.active, + ampl_vary.active, + slope_vary.active, + offset_vary.active, + ], + constraints_min=[ + centre_min.value, + sigma_min.value, + ampl_min.value, + slope_min.value, + offset_min.value, + ], + constraints_max=[ + centre_max.value, + sigma_max.value, + ampl_max.value, + slope_max.value, + offset_max.value, + ], + ) + + _update_plot(meas) + + fit_button = Button(label="Fit Current") + fit_button.on_click(fit_button_callback) + def export_results(det_data): if det_data["meta"]["indices"] == "hkl": ext = ".comm" @@ -317,6 +382,7 @@ def create(): row(peak_int_ratio_spinner, peak_prominence_spinner), smooth_toggle, row(window_size_spinner, poly_order_spinner), + peakfind_button, peakfind_all_button, ) @@ -347,6 +413,7 @@ def create(): column(div_8, slope_guess, slope_vary, slope_min, slope_max), column(div_9, offset_guess, offset_vary, offset_min, offset_max), ), + row(fit_button), row(fit_all_button), ) From 56708826147dd19eb3aca7432b43c4f911b60c72 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 6 Oct 2020 11:53:08 +0200 Subject: [PATCH 064/116] Display fit results in concise way --- pyzebra/app/panel_ccl_integrate.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index d3d3c52..116ad31 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -127,7 +127,28 @@ def create(): if fit is not None: plot_gauss_source.data.update(x=x, y=meas["fit"]["comps"]["gaussian"]) plot_bkg_source.data.update(x=x, y=meas["fit"]["comps"]["background"]) - fit_output_textinput.value = str(fit["full_report"]) + params = fit["result"].params + fit_output_textinput.value = ( + "%s \n" + "Gaussian: centre = %9.4f, sigma = %9.4f, area = %9.4f \n" + "background: slope = %9.4f, intercept = %9.4f \n" + "Int. area = %9.4f +/- %9.4f \n" + "fit area = %9.4f +/- %9.4f \n" + "ratio((fit-int)/fit) = %9.4f" + % ( + ind, + params["g_cen"].value, + params["g_width"].value, + params["g_amp"].value, + params["slope"].value, + params["intercept"].value, + fit["int_area"].n, + fit["int_area"].s, + params["g_amp"].value, + params["g_amp"].stderr, + (params["g_amp"].value - fit["int_area"].n) / params["g_amp"].value, + ) + ) else: plot_gauss_source.data.update(x=[], y=[]) plot_bkg_source.data.update(x=[], y=[]) From df546cb688fffc9bf4a84533b0af99bf292331cc Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 6 Oct 2020 13:21:39 +0200 Subject: [PATCH 065/116] Add fit param reset button --- pyzebra/app/panel_ccl_integrate.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 116ad31..722e3e5 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -243,6 +243,31 @@ def create(): offset_min = Spinner(default_size=100) offset_max = Spinner(default_size=100) + def fitparam_reset_button_callback(): + centre_guess.value = None + centre_vary.active = True + centre_min.value = None + centre_max.value = None + sigma_guess.value = None + sigma_vary.active = True + sigma_min.value = None + sigma_max.value = None + ampl_guess.value = None + ampl_vary.active = True + ampl_min.value = None + ampl_max.value = None + slope_guess.value = None + slope_vary.active = True + slope_min.value = None + slope_max.value = None + offset_guess.value = None + offset_vary.active = True + offset_min.value = None + offset_max.value = None + + fitparam_reset_button = Button(label="Reset to defaults") + fitparam_reset_button.on_click(fitparam_reset_button_callback) + fit_output_textinput = TextAreaInput(title="Fit results:", width=600, height=400) def peakfind_all_button_callback(): @@ -434,6 +459,7 @@ def create(): column(div_8, slope_guess, slope_vary, slope_min, slope_max), column(div_9, offset_guess, offset_vary, offset_min, offset_max), ), + row(fitparam_reset_button), row(fit_button), row(fit_all_button), ) From bd4e5980ab0be10c1993565803aec3285548c9d9 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 6 Oct 2020 13:28:25 +0200 Subject: [PATCH 066/116] Display smoothed curve --- pyzebra/app/panel_ccl_integrate.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 722e3e5..64b7e5d 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -117,9 +117,11 @@ def create(): peak_pos_textinput.value = str(peak_indexes) plot_circle_source.data.update(x=peak_indexes, y=meas["peak_heights"]) + plot_line_smooth_source.data.update(x=x, y=meas["smooth_peaks"]) else: peak_pos_textinput.value = None plot_circle_source.data.update(x=[], y=[]) + plot_line_smooth_source.data.update(x=[], y=[]) peak_pos_textinput_lock = False @@ -172,6 +174,11 @@ def create(): plot_line_source = ColumnDataSource(dict(x=[0], y=[0])) plot.add_glyph(plot_line_source, Line(x="x", y="y", line_color="steelblue")) + plot_line_smooth_source = ColumnDataSource(dict(x=[0], y=[0])) + plot.add_glyph( + plot_line_smooth_source, Line(x="x", y="y", line_color="steelblue", line_dash="dashed") + ) + plot_gauss_source = ColumnDataSource(dict(x=[0], y=[0])) plot.add_glyph(plot_gauss_source, Line(x="x", y="y", line_color="red", line_dash="dashed")) From e1fbe075fea87194f83f47489e26f59ec6780b32 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 6 Oct 2020 15:47:20 +0200 Subject: [PATCH 067/116] Return preview of an export file --- pyzebra/app/panel_ccl_integrate.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 64b7e5d..c3ad3ae 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -275,7 +275,7 @@ def create(): fitparam_reset_button = Button(label="Reset to defaults") fitparam_reset_button.on_click(fitparam_reset_button_callback) - fit_output_textinput = TextAreaInput(title="Fit results:", width=600, height=400) + fit_output_textinput = TextAreaInput(title="Fit results:", width=450, height=400) def peakfind_all_button_callback(): nonlocal det_data @@ -407,6 +407,24 @@ def create(): fit_button = Button(label="Fit Current") fit_button.on_click(fit_button_callback) + preview_output_textinput = TextAreaInput(title="Export file preview:", width=450, height=400) + + def preview_output_button_callback(): + if det_data["meta"]["indices"] == "hkl": + ext = ".comm" + elif det_data["meta"]["indices"] == "real": + ext = ".incomm" + + with tempfile.TemporaryDirectory() as temp_dir: + temp_file = temp_dir + "/temp" + pyzebra.export_comm(det_data, temp_file) + + with open(f"{temp_file}{ext}") as f: + preview_output_textinput.value = f.read() + + preview_output_button = Button(label="Preview file", default_size=220) + preview_output_button.on_click(preview_output_button_callback) + def export_results(det_data): if det_data["meta"]["indices"] == "hkl": ext = ".comm" @@ -426,7 +444,7 @@ def create(): cont, ext = export_results(det_data) js_data.data.update(cont=[cont], ext=[ext]) - save_button = Button(label="Export to .comm/.incomm file:") + save_button = Button(label="Download file", button_type="success", default_size=220) save_button.on_click(save_button_callback) save_button.js_on_click(CustomJS(args={"js_data": js_data}, code=javaScript)) @@ -471,12 +489,14 @@ def create(): row(fit_all_button), ) + export_layout = column(preview_output_textinput, row(preview_output_button, save_button)) + upload_div = Div(text="Or upload .ccl file:") tab_layout = column( row(proposal_textinput, ccl_file_select), row(column(Spacer(height=5), upload_div), upload_button), - row(meas_table, plot, Spacer(width=30), fit_output_textinput), - row(findpeak_controls, Spacer(width=30), fitpeak_controls, column(save_button)), + row(meas_table, plot, Spacer(width=30), fit_output_textinput, export_layout), + row(findpeak_controls, Spacer(width=30), fitpeak_controls), ) return Panel(child=tab_layout, title="ccl integrate") From b4a0f43695054d42768b9222d8f52b49e2eda79f Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Tue, 6 Oct 2020 16:04:50 +0200 Subject: [PATCH 068/116] added more functions --- pyzebra/ccl_dict_operation.py | 312 ++++++++++++++++++++++------------ 1 file changed, 200 insertions(+), 112 deletions(-) diff --git a/pyzebra/ccl_dict_operation.py b/pyzebra/ccl_dict_operation.py index 6855e14..eb8e792 100644 --- a/pyzebra/ccl_dict_operation.py +++ b/pyzebra/ccl_dict_operation.py @@ -1,6 +1,73 @@ import numpy as np import uncertainties as u from fitvol2 import create_uncertanities +import matplotlib.pyplot as plt + + +def add_dict(dict1, dict2): + """adds two dictionaries, meta of the new is saved as meata+original_filename and + measurements are shifted to continue with numbering of first dict + :arg dict1 : dictionarry to add to + :arg dict2 : dictionarry from which to take the measurements + :return dict1 : combined dictionary + Note: dict1 must be made from ccl, otherwise we would have to change the structure of loaded + dat file""" + max_measurement_dict1 = max([int(str(keys)[1:]) for keys in dict1["Measurements"]]) + if dict2['meta']['data_type'] == '.ccl': + new_filenames = ['M' + str(x + max_measurement_dict1) for x in + [int(str(keys)[1:]) for keys in dict2["Measurements"]]] + new_meta_name = 'meta' + str(dict2["meta"]['original_filename']) + if new_meta_name not in dict1: + for keys, name in zip(dict2["Measurements"], new_filenames): + dict2["Measurements"][keys]['file_of_origin'] = str(dict2["meta"]['original_filename']) + dict1["Measurements"][name] = dict2["Measurements"][keys] + + dict1[new_meta_name] = dict2["meta"] + + + else: + raise KeyError(str('The file %s has alredy been added to %s' % (dict2["meta"][ + 'original_filename'], + dict1["meta"][ + 'original_filename']))) + elif dict2['meta']['data_type'] == '.dat': + d = {} + new_name = 'M' + str(max_measurement_dict1+1) + hkl = dict2['meta']['title'] + d['h_index'] = float(hkl.split()[-3]) + d['k_index'] = float(hkl.split()[-2]) + d['l_index'] = float(hkl.split()[-1]) + d['number_of_measurements'] = len(dict2['Measurements']['NP']) + d['om'] = dict2['Measurements']['om'] + d['Counts'] = dict2['Measurements']['Counts'] + d['monitor'] = dict2['Measurements']['Monitor1'][0] + d['temperature'] = dict2['meta']['temp'] + d['mag_field'] = dict2['meta']['mf'] + d['omega_angle'] = dict2['meta']['omega'] + dict1["Measurements"][new_name] = d + print(hkl.split()) + for keys in d: + print(keys) + + + + print('s') + + return dict1 + +def auto(dict): + """takes just unique tuples from all tuples in dictionary returend by scan_dict + intendet for automatic merge if you doesent want to specify what scans to merge together + args: dict - dictionary from scan_dict function + :return dict - dict without repetitions""" + for keys in dict: + tuple_list = dict[keys] + new = list() + for i in range(len(tuple_list)): + if tuple_list[0][0] == tuple_list[i][0]: + new.append(tuple_list[i]) + dict[keys] = new + return dict def scan_dict(dict): @@ -9,8 +76,8 @@ def scan_dict(dict): :return dictionary with matching scans, if there are none, the dict is empty note: can be checked by "not d", true if empty """ + d = {} - dupl = 0 for i in dict["Measurements"]: for j in dict["Measurements"]: if dict["Measurements"][str(i)] != dict["Measurements"][str(j)]: @@ -27,48 +94,49 @@ def scan_dict(dict): if itup != jtup: pass else: + if str(itup) not in d: - d[str(itup)] = (i, j) + d[str(itup)] = list() + d[str(itup)].append((i, j)) else: - dupl = dupl + 1 - d[str(itup) + "_" + str(dupl)] = (i, j) + d[str(itup)].append((i, j)) else: continue return d def compare_hkl(dict1, dict2): - """Compares two dictionaries based on hkl indexes and return dictionary with str(h k l) as + """Compares two dictionaries based on hkl indexes and return dictionary with str(h k l) as key and tuple with keys to same measurement in dict1 and dict2 :arg dict1 : first dictionary :arg dict2 : second dictionary :return d : dict with matches - example of one key: '0.0 0.0 -1.0 : ('M1', 'M9')' meaning that 001 hkl measurement is M1 in + example of one key: '0.0 0.0 -1.0 : ('M1', 'M9')' meaning that 001 hkl measurement is M1 in first dict and M9 in second""" d = {} dupl = 0 for keys in dict1["Measurements"]: for key in dict2["Measurements"]: if ( - dict1["Measurements"][str(keys)]["h_index"] - == dict2["Measurements"][str(key)]["h_index"] - and dict1["Measurements"][str(keys)]["k_index"] - == dict2["Measurements"][str(key)]["k_index"] - and dict1["Measurements"][str(keys)]["l_index"] - == dict2["Measurements"][str(key)]["l_index"] + dict1["Measurements"][str(keys)]["h_index"] + == dict2["Measurements"][str(key)]["h_index"] + and dict1["Measurements"][str(keys)]["k_index"] + == dict2["Measurements"][str(key)]["k_index"] + and dict1["Measurements"][str(keys)]["l_index"] + == dict2["Measurements"][str(key)]["l_index"] ): if ( - str( - ( - str(dict1["Measurements"][str(keys)]["h_index"]) - + " " - + str(dict1["Measurements"][str(keys)]["k_index"]) - + " " - + str(dict1["Measurements"][str(keys)]["l_index"]) + str( + ( + str(dict1["Measurements"][str(keys)]["h_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["k_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["l_index"]) + ) ) - ) - not in d + not in d ): d[ str( @@ -103,7 +171,7 @@ def create_tuples(x, y, y_err): Counts need to be normalized to monitor before""" t = list() for i in range(len(x)): - tup = (x[i], y[i], y_err) + tup = (x[i], y[i], y_err[i]) t.append(tup) return t @@ -149,7 +217,7 @@ def merge(dict1, dict2, keys, auto=True, monitor=100000): cor_y2, y_err2 = normalize(dict2, keys[1], monitor=monitor) # creates touples (om, Counts, sigma) for sorting and further processing tuple_list = create_tuples(x1, cor_y1, y_err1) + create_tuples(x2, cor_y2, y_err2) - # Sort the list on om and add 0 0 0 tuple to the last postion + # Sort the list on om and add 0 0 0 tuple to the last position sorted_t = sorted(tuple_list, key=lambda tup: tup[0]) sorted_t.append((0, 0, 0)) om, Counts, sigma = [], [], [] @@ -187,6 +255,7 @@ def merge(dict1, dict2, keys, auto=True, monitor=100000): dict1["Measurements"][keys[0]]["Counts"] = Counts dict1["Measurements"][keys[0]]["sigma"] = sigma dict1["Measurements"][keys[0]]["monitor"] = monitor + print('merging done') return dict1 @@ -219,10 +288,17 @@ def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): for k in range(len(res)): res_nom = np.append(res_nom, res[k].n) res_err = np.append(res_err, res[k].s) - - if len([num for num in res_nom if num < 0]) >= 0.3*len(res_nom): + # plt.plot(res_nom, 'r', label='final') + # plt.plot(cor_y1, 'g', label='dict1') + # plt.plot(cor_y2, 'b', label='dict2') + # plt.legend() + # plt.title(str(keys)) + # plt.show() + + if len([num for num in res_nom if num < 0]) >= 0.3 * len(res_nom): print(f'Warning! percentage of negative numbers in measurement subsracted {keys[0]} is ' - f'{len([num for num in res_nom if num < 0])/len(res_nom)}') + f'{len([num for num in res_nom if num < 0]) / len(res_nom)}') + dict1["Measurements"][str(keys[0])]["Counts"] = res_nom dict1["Measurements"][str(keys[0])]["sigma"] = res_err dict1["Measurements"][str(keys[0])]["monitor"] = monitor @@ -243,7 +319,7 @@ def compare_dict(dict1, dict2): information of how many measurement differ and which ones differ :returns report_string string comparing all different values respecively of measurements""" - if dict1["file_type"] != dict2["file_type"]: + if dict1["meta"]["data_type"] != dict2["meta"]["data_type"]: print("select two dicts") return S = [] @@ -283,16 +359,19 @@ def compare_dict(dict1, dict2): for k in dict1["meta"] if k in dict2["meta"] and dict1["meta"][k] != dict2["meta"][k] } - + exlude_meta_set = ["original_filename", "date", "title"] for keys in different_meta: - if keys not in conflicts: - conflicts[keys] = 1 + if keys in exlude_meta_set: + continue else: - conflicts[keys] = conflicts[keys] + 1 + if keys not in conflicts: + conflicts[keys] = 1 + else: + conflicts[keys] = conflicts[keys] + 1 - S.append(" Different values in %s \n" % str(keys)) - S.append(" dict1: %s \n" % str(dict1["meta"][str(keys)])) - S.append(" dict2: %s \n" % str(dict2["meta"][str(keys)])) + S.append(" Different values in %s \n" % str(keys)) + S.append(" dict1: %s \n" % str(dict1["meta"][str(keys)])) + S.append(" dict2: %s \n" % str(dict2["meta"][str(keys)])) # compare Measurements S.append( @@ -305,80 +384,89 @@ def compare_dict(dict1, dict2): ) S.append("Different values in Measurements:\n") select_set = ["om", "Counts", "sigma"] - exlude_set = ["time", "Counts"] - + exlude_set = ["time", "Counts", "date", 'notes'] for keys1 in comp: for key2 in dict1["Measurements"][str(comp[str(keys1)][0])]: if key2 in exlude_set: continue if key2 not in select_set: - if ( - dict1["Measurements"][comp[str(keys1)][0]][str(key2)] - != dict2["Measurements"][str(comp[str(keys1)][0])][str(key2)] - ): - S.append( - "Measurement value " - "%s" - ", with hkl %s differs in meausrements %s and %s \n" - % (key2, keys1, comp[str(keys1)][0], comp[str(keys1)][1]) - ) - S.append( - " dict1: %s \n" - % str(dict1["Measurements"][comp[str(keys1)][0]][str(key2)]) - ) - S.append( - " dict2: %s \n" - % str(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) - ) - if key2 not in conflicts: - conflicts[key2] = {} - conflicts[key2]["amount"] = 1 - conflicts[key2]["measurements"] = str(comp[str(keys1)][0]) - else: - conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 - conflicts[key2]["measurements"] = ( - conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)][0])) + try: + if ( + dict1["Measurements"][comp[str(keys1)][0]][str(key2)] + != dict2["Measurements"][str(comp[str(keys1)][1])][str(key2)] + ): + S.append( + "Measurement value " + "%s" + ", with hkl %s differs in meausrements %s and %s \n" + % (key2, keys1, comp[str(keys1)][0], comp[str(keys1)][1]) ) + S.append( + " dict1: %s \n" + % str(dict1["Measurements"][comp[str(keys1)][0]][str(key2)]) + ) + S.append( + " dict2: %s \n" + % str(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) + ) + if key2 not in conflicts: + conflicts[key2] = {} + conflicts[key2]["amount"] = 1 + conflicts[key2]["measurements"] = str(comp[str(keys1)]) + else: + + conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 + conflicts[key2]["measurements"] = ( + conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)])) + ) + except KeyError as e: + print('Missing keys, some files were probably merged or substracted') + print(e.args) + else: - comparison = list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)]) == list( - dict2["Measurements"][comp[str(keys1)][1]][str(key2)] - ) - if len(list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)])) != len( - list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) - ): - if str("different length of %s" % key2) not in warnings: - warnings[str("different length of %s" % key2)] = list() - warnings[str("different length of %s" % key2)].append( - (str(comp[keys1][0]), str(comp[keys1][1])) - ) - else: - warnings[str("different length of %s" % key2)].append( - (str(comp[keys1][0]), str(comp[keys1][1])) - ) - if not comparison: - S.append( - "Measurement value " - "%s" - " differs in measurement %s and %s \n" - % (key2, comp[str(keys1)][0], comp[str(keys1)][1]) + try: + comparison = list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)]) == list( + dict2["Measurements"][comp[str(keys1)][1]][str(key2)] ) - S.append( - " dict1: %s \n" - % str(list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)])) - ) - S.append( - " dict2: %s \n" - % str(list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)])) - ) - if key2 not in conflicts: - conflicts[key2] = {} - conflicts[key2]["amount"] = 1 - conflicts[key2]["measurements"] = str(comp[str(keys1)][0]) - else: - conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 - conflicts[key2]["measurements"] = ( - conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)][1])) + if len(list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)])) != len( + list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) + ): + if str("different length of %s" % key2) not in warnings: + warnings[str("different length of %s" % key2)] = list() + warnings[str("different length of %s" % key2)].append( + (str(comp[keys1][0]), str(comp[keys1][1])) + ) + else: + warnings[str("different length of %s" % key2)].append( + (str(comp[keys1][0]), str(comp[keys1][1])) + ) + if not comparison: + S.append( + "Measurement value " + "%s" + " differs in measurement %s and %s \n" + % (key2, comp[str(keys1)][0], comp[str(keys1)][1]) ) + S.append( + " dict1: %s \n" + % str(list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)])) + ) + S.append( + " dict2: %s \n" + % str(list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)])) + ) + if key2 not in conflicts: + conflicts[key2] = {} + conflicts[key2]["amount"] = 1 + conflicts[key2]["measurements"] = str(comp[str(keys1)]) + else: + conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 + conflicts[key2]["measurements"] = ( + conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)])) + ) + except KeyError as e: + print('Missing keys, some files were probably merged or substracted') + print(e.args) for keys in conflicts: try: @@ -390,27 +478,27 @@ def compare_dict(dict1, dict2): def guess_next(dict1, dict2, comp): - """iterates thorough the scans and tries to decide if the scans should be + """iterates thorough the scans and tries to decide if the scans should be substracted or merged""" threshold = 0.05 for keys in comp: if ( - abs( - ( - dict1["Measurements"][str(comp[keys][0])]["temperature"] - - dict2["Measurements"][str(comp[keys][1])]["temperature"] + abs( + ( + dict1["Measurements"][str(comp[keys][0])]["temperature"] + - dict2["Measurements"][str(comp[keys][1])]["temperature"] + ) + / dict2["Measurements"][str(comp[keys][1])]["temperature"] ) - / dict2["Measurements"][str(comp[keys][1])]["temperature"] - ) - < threshold - and abs( - ( + < threshold + and abs( + ( dict1["Measurements"][str(comp[keys][0])]["mag_field"] - dict2["Measurements"][str(comp[keys][1])]["mag_field"] - ) - / dict2["Measurements"][str(comp[keys][1])]["mag_field"] ) - < threshold + / dict2["Measurements"][str(comp[keys][1])]["mag_field"] + ) + < threshold ): comp[keys] = comp[keys] + tuple("m") else: From ca5579ddd97fca6694addd373e383bc6ac54c498 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Tue, 6 Oct 2020 16:47:00 +0200 Subject: [PATCH 069/116] Update fit2.py Added check for stderr of fit area which can be None in case fit failure and can break the rest of the code. --- pyzebra/fit2.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 8468c62..2b96a80 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -142,6 +142,11 @@ def fitccl( # u.ufloat to work with uncertanities fit_area = u.ufloat(result.params["g_amp"].value, result.params["g_amp"].stderr) comps = result.eval_components() + + if result.params["g_amp"].stderr is None: + result.params["g_amp"].stderr = result.params["g_amp"].value + elif result.params["g_amp"].stderr > result.params["g_amp"].value: + result.params["g_amp"].stderr = result.params["g_amp"].value if len(meas["peak_indexes"]) == 0: # for case of no peak, there is no reason to integrate, therefore fit and int are equal From 7a50f2c8d539ca7112313ace01559740791216d1 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Wed, 7 Oct 2020 10:29:04 +0200 Subject: [PATCH 070/116] Update fit2.py Added user input to no peak case --- pyzebra/fit2.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 2b96a80..bb7e359 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -65,11 +65,11 @@ def fitccl( # Case for no peak, gaussian in centre, sigma as 20% of range print("No peak") peak_index = find_nearest(x, np.mean(x)) - guess[0] = x[int(peak_index)] - guess[1] = (x[-1] - x[0]) / 5 - guess[2] = 10 - guess[3] = 0 - guess[4] = np.mean(y) + guess[0] = x[int(peak_index)] if guess[0] is None else guess[0] + guess[1] = (x[-1] - x[0])/5 if guess[1] is None else guess[1] + guess[2] = 50 if guess[2] is None else guess[2] + guess[3] = 0 if guess[3] is None else guess[3] + guess[4] = np.mean(y) if guess[4] is None else guess[4] constraints_min[2] = 0 elif len(meas["peak_indexes"]) == 1: From e2f00a3bcef5a30eb16bde18f27a60440e7f032c Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Wed, 7 Oct 2020 16:16:37 +0200 Subject: [PATCH 071/116] Fix imports and apply formatter --- pyzebra/__init__.py | 1 + pyzebra/ccl_dict_operation.py | 161 +++++++++++++++++----------------- 2 files changed, 83 insertions(+), 79 deletions(-) diff --git a/pyzebra/__init__.py b/pyzebra/__init__.py index 469fd9f..847d3fa 100644 --- a/pyzebra/__init__.py +++ b/pyzebra/__init__.py @@ -5,3 +5,4 @@ from pyzebra.load_1D import load_1D, parse_1D from pyzebra.ccl_findpeaks import ccl_findpeaks from pyzebra.fit2 import fitccl from pyzebra.comm_export import export_comm +import pyzebra.ccl_dict_operation diff --git a/pyzebra/ccl_dict_operation.py b/pyzebra/ccl_dict_operation.py index eb8e792..755a4af 100644 --- a/pyzebra/ccl_dict_operation.py +++ b/pyzebra/ccl_dict_operation.py @@ -1,7 +1,6 @@ import numpy as np import uncertainties as u -from fitvol2 import create_uncertanities -import matplotlib.pyplot as plt +from .fit2 import create_uncertanities def add_dict(dict1, dict2): @@ -13,48 +12,52 @@ def add_dict(dict1, dict2): Note: dict1 must be made from ccl, otherwise we would have to change the structure of loaded dat file""" max_measurement_dict1 = max([int(str(keys)[1:]) for keys in dict1["Measurements"]]) - if dict2['meta']['data_type'] == '.ccl': - new_filenames = ['M' + str(x + max_measurement_dict1) for x in - [int(str(keys)[1:]) for keys in dict2["Measurements"]]] - new_meta_name = 'meta' + str(dict2["meta"]['original_filename']) + if dict2["meta"]["data_type"] == ".ccl": + new_filenames = [ + "M" + str(x + max_measurement_dict1) + for x in [int(str(keys)[1:]) for keys in dict2["Measurements"]] + ] + new_meta_name = "meta" + str(dict2["meta"]["original_filename"]) if new_meta_name not in dict1: for keys, name in zip(dict2["Measurements"], new_filenames): - dict2["Measurements"][keys]['file_of_origin'] = str(dict2["meta"]['original_filename']) + dict2["Measurements"][keys]["file_of_origin"] = str( + dict2["meta"]["original_filename"] + ) dict1["Measurements"][name] = dict2["Measurements"][keys] dict1[new_meta_name] = dict2["meta"] - else: - raise KeyError(str('The file %s has alredy been added to %s' % (dict2["meta"][ - 'original_filename'], - dict1["meta"][ - 'original_filename']))) - elif dict2['meta']['data_type'] == '.dat': + raise KeyError( + str( + "The file %s has alredy been added to %s" + % (dict2["meta"]["original_filename"], dict1["meta"]["original_filename"]) + ) + ) + elif dict2["meta"]["data_type"] == ".dat": d = {} - new_name = 'M' + str(max_measurement_dict1+1) - hkl = dict2['meta']['title'] - d['h_index'] = float(hkl.split()[-3]) - d['k_index'] = float(hkl.split()[-2]) - d['l_index'] = float(hkl.split()[-1]) - d['number_of_measurements'] = len(dict2['Measurements']['NP']) - d['om'] = dict2['Measurements']['om'] - d['Counts'] = dict2['Measurements']['Counts'] - d['monitor'] = dict2['Measurements']['Monitor1'][0] - d['temperature'] = dict2['meta']['temp'] - d['mag_field'] = dict2['meta']['mf'] - d['omega_angle'] = dict2['meta']['omega'] + new_name = "M" + str(max_measurement_dict1 + 1) + hkl = dict2["meta"]["title"] + d["h_index"] = float(hkl.split()[-3]) + d["k_index"] = float(hkl.split()[-2]) + d["l_index"] = float(hkl.split()[-1]) + d["number_of_measurements"] = len(dict2["Measurements"]["NP"]) + d["om"] = dict2["Measurements"]["om"] + d["Counts"] = dict2["Measurements"]["Counts"] + d["monitor"] = dict2["Measurements"]["Monitor1"][0] + d["temperature"] = dict2["meta"]["temp"] + d["mag_field"] = dict2["meta"]["mf"] + d["omega_angle"] = dict2["meta"]["omega"] dict1["Measurements"][new_name] = d print(hkl.split()) for keys in d: print(keys) - - - print('s') + print("s") return dict1 + def auto(dict): """takes just unique tuples from all tuples in dictionary returend by scan_dict intendet for automatic merge if you doesent want to specify what scans to merge together @@ -118,25 +121,25 @@ def compare_hkl(dict1, dict2): for keys in dict1["Measurements"]: for key in dict2["Measurements"]: if ( - dict1["Measurements"][str(keys)]["h_index"] - == dict2["Measurements"][str(key)]["h_index"] - and dict1["Measurements"][str(keys)]["k_index"] - == dict2["Measurements"][str(key)]["k_index"] - and dict1["Measurements"][str(keys)]["l_index"] - == dict2["Measurements"][str(key)]["l_index"] + dict1["Measurements"][str(keys)]["h_index"] + == dict2["Measurements"][str(key)]["h_index"] + and dict1["Measurements"][str(keys)]["k_index"] + == dict2["Measurements"][str(key)]["k_index"] + and dict1["Measurements"][str(keys)]["l_index"] + == dict2["Measurements"][str(key)]["l_index"] ): if ( - str( - ( - str(dict1["Measurements"][str(keys)]["h_index"]) - + " " - + str(dict1["Measurements"][str(keys)]["k_index"]) - + " " - + str(dict1["Measurements"][str(keys)]["l_index"]) - ) + str( + ( + str(dict1["Measurements"][str(keys)]["h_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["k_index"]) + + " " + + str(dict1["Measurements"][str(keys)]["l_index"]) ) - not in d + ) + not in d ): d[ str( @@ -244,8 +247,10 @@ def merge(dict1, dict2, keys, auto=True, monitor=100000): if dict1 == dict2: del dict1["Measurements"][keys[1]] - note = f'This measurement was merged with measurement {keys[1]} from ' \ - f'file {dict2["meta"]["original_filename"]} \n' + note = ( + f"This measurement was merged with measurement {keys[1]} from " + f'file {dict2["meta"]["original_filename"]} \n' + ) if "notes" not in dict1["Measurements"][str(keys[0])]: dict1["Measurements"][str(keys[0])]["notes"] = note else: @@ -255,7 +260,7 @@ def merge(dict1, dict2, keys, auto=True, monitor=100000): dict1["Measurements"][keys[0]]["Counts"] = Counts dict1["Measurements"][keys[0]]["sigma"] = sigma dict1["Measurements"][keys[0]]["monitor"] = monitor - print('merging done') + print("merging done") return dict1 @@ -288,22 +293,20 @@ def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): for k in range(len(res)): res_nom = np.append(res_nom, res[k].n) res_err = np.append(res_err, res[k].s) - # plt.plot(res_nom, 'r', label='final') - # plt.plot(cor_y1, 'g', label='dict1') - # plt.plot(cor_y2, 'b', label='dict2') - # plt.legend() - # plt.title(str(keys)) - # plt.show() if len([num for num in res_nom if num < 0]) >= 0.3 * len(res_nom): - print(f'Warning! percentage of negative numbers in measurement subsracted {keys[0]} is ' - f'{len([num for num in res_nom if num < 0]) / len(res_nom)}') + print( + f"Warning! percentage of negative numbers in measurement subsracted {keys[0]} is " + f"{len([num for num in res_nom if num < 0]) / len(res_nom)}" + ) dict1["Measurements"][str(keys[0])]["Counts"] = res_nom dict1["Measurements"][str(keys[0])]["sigma"] = res_err dict1["Measurements"][str(keys[0])]["monitor"] = monitor - note = f'Measurement {keys[1]} from file {dict2["meta"]["original_filename"]} ' \ - f'was substracted from this measurement \n' + note = ( + f'Measurement {keys[1]} from file {dict2["meta"]["original_filename"]} ' + f"was substracted from this measurement \n" + ) if "notes" not in dict1["Measurements"][str(keys[0])]: dict1["Measurements"][str(keys[0])]["notes"] = note else: @@ -384,7 +387,7 @@ def compare_dict(dict1, dict2): ) S.append("Different values in Measurements:\n") select_set = ["om", "Counts", "sigma"] - exlude_set = ["time", "Counts", "date", 'notes'] + exlude_set = ["time", "Counts", "date", "notes"] for keys1 in comp: for key2 in dict1["Measurements"][str(comp[str(keys1)][0])]: if key2 in exlude_set: @@ -392,8 +395,8 @@ def compare_dict(dict1, dict2): if key2 not in select_set: try: if ( - dict1["Measurements"][comp[str(keys1)][0]][str(key2)] - != dict2["Measurements"][str(comp[str(keys1)][1])][str(key2)] + dict1["Measurements"][comp[str(keys1)][0]][str(key2)] + != dict2["Measurements"][str(comp[str(keys1)][1])][str(key2)] ): S.append( "Measurement value " @@ -417,19 +420,19 @@ def compare_dict(dict1, dict2): conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 conflicts[key2]["measurements"] = ( - conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)])) + conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)])) ) except KeyError as e: - print('Missing keys, some files were probably merged or substracted') + print("Missing keys, some files were probably merged or substracted") print(e.args) else: try: - comparison = list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)]) == list( - dict2["Measurements"][comp[str(keys1)][1]][str(key2)] - ) + comparison = list( + dict1["Measurements"][comp[str(keys1)][0]][str(key2)] + ) == list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) if len(list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)])) != len( - list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) + list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) ): if str("different length of %s" % key2) not in warnings: warnings[str("different length of %s" % key2)] = list() @@ -462,10 +465,10 @@ def compare_dict(dict1, dict2): else: conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 conflicts[key2]["measurements"] = ( - conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)])) + conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)])) ) except KeyError as e: - print('Missing keys, some files were probably merged or substracted') + print("Missing keys, some files were probably merged or substracted") print(e.args) for keys in conflicts: @@ -483,22 +486,22 @@ def guess_next(dict1, dict2, comp): threshold = 0.05 for keys in comp: if ( - abs( - ( - dict1["Measurements"][str(comp[keys][0])]["temperature"] - - dict2["Measurements"][str(comp[keys][1])]["temperature"] - ) - / dict2["Measurements"][str(comp[keys][1])]["temperature"] + abs( + ( + dict1["Measurements"][str(comp[keys][0])]["temperature"] + - dict2["Measurements"][str(comp[keys][1])]["temperature"] ) - < threshold - and abs( - ( + / dict2["Measurements"][str(comp[keys][1])]["temperature"] + ) + < threshold + and abs( + ( dict1["Measurements"][str(comp[keys][0])]["mag_field"] - dict2["Measurements"][str(comp[keys][1])]["mag_field"] + ) + / dict2["Measurements"][str(comp[keys][1])]["mag_field"] ) - / dict2["Measurements"][str(comp[keys][1])]["mag_field"] - ) - < threshold + < threshold ): comp[keys] = comp[keys] + tuple("m") else: From 55c3a9bce116a6db00917c2e2284bd44352b70f6 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 8 Oct 2020 13:09:18 +0200 Subject: [PATCH 072/116] Parse peak_indexes as int --- pyzebra/app/panel_ccl_integrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index c3ad3ae..166d978 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -213,7 +213,7 @@ def create(): meas = det_data["Measurements"][meas_name] meas["num_of_peaks"] = 1 - meas["peak_indexes"] = [float(new)] + meas["peak_indexes"] = [int(new)] meas["peak_heights"] = [0] _update_table() _update_plot(meas_name) From e679ccdce4d499907c9ec4fedb6103f1fca2d541 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 8 Oct 2020 15:08:54 +0200 Subject: [PATCH 073/116] Add export_fit checkbox --- pyzebra/app/panel_ccl_integrate.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 166d978..1e877db 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -7,6 +7,7 @@ from bokeh.layouts import column, row from bokeh.models import ( BasicTicker, Button, + CheckboxGroup, Circle, ColumnDataSource, CustomJS, @@ -127,6 +128,11 @@ def create(): fit = meas.get("fit") if fit is not None: + if meas["fit"]["export_fit"]: + export_fit_checkbox.active = [0] + else: + export_fit_checkbox.active = [] + plot_gauss_source.data.update(x=x, y=meas["fit"]["comps"]["gaussian"]) plot_bkg_source.data.update(x=x, y=meas["fit"]["comps"]["background"]) params = fit["result"].params @@ -407,6 +413,14 @@ def create(): fit_button = Button(label="Fit Current") fit_button.on_click(fit_button_callback) + def export_fit_checkbox_callback(_attr, _old, new): + sel_ind = meas_table_source.selected.indices[-1] + meas = meas_table_source.data["measurement"][sel_ind] + det_data["Measurements"][meas]["fit"]["export_fit"] = bool(new) + + export_fit_checkbox = CheckboxGroup(labels=["Export fit"], width=100) + export_fit_checkbox.on_change("active", export_fit_checkbox_callback) + preview_output_textinput = TextAreaInput(title="Export file preview:", width=450, height=400) def preview_output_button_callback(): @@ -484,7 +498,7 @@ def create(): column(div_8, slope_guess, slope_vary, slope_min, slope_max), column(div_9, offset_guess, offset_vary, offset_min, offset_max), ), - row(fitparam_reset_button), + row(fitparam_reset_button, export_fit_checkbox), row(fit_button), row(fit_all_button), ) From 94551acc64ac79ae58b3fc34ac73d9925985edff Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 8 Oct 2020 16:31:14 +0200 Subject: [PATCH 074/116] Add integrate from .. to .. widgets --- pyzebra/app/panel_ccl_integrate.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 1e877db..97fb970 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -255,6 +255,8 @@ def create(): offset_vary = Toggle(default_size=100, active=True) offset_min = Spinner(default_size=100) offset_max = Spinner(default_size=100) + integ_from = Spinner(title="Integrate from:", default_size=145) + integ_to = Spinner(title="to:", default_size=145) def fitparam_reset_button_callback(): centre_guess.value = None @@ -277,6 +279,8 @@ def create(): offset_vary.active = True offset_min.value = None offset_max.value = None + integ_from.value = None + integ_to.value = None fitparam_reset_button = Button(label="Reset to defaults") fitparam_reset_button.on_click(fitparam_reset_button_callback) @@ -360,6 +364,8 @@ def create(): slope_max.value, offset_max.value, ], + numfit_min=integ_from.value, + numfit_max=integ_to.value, ) sel_ind = meas_table_source.selected.indices[-1] @@ -498,7 +504,8 @@ def create(): column(div_8, slope_guess, slope_vary, slope_min, slope_max), column(div_9, offset_guess, offset_vary, offset_min, offset_max), ), - row(fitparam_reset_button, export_fit_checkbox), + row(integ_from, integ_to, column(Spacer(height=25), export_fit_checkbox)), + row(fitparam_reset_button), row(fit_button), row(fit_all_button), ) From d0d993a8139586687aac022a249098d98ee4f08e Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Fri, 9 Oct 2020 11:26:27 +0200 Subject: [PATCH 075/116] Update fit2.py added option for data binning --- pyzebra/fit2.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index bb7e359..6d68a09 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -1,9 +1,15 @@ import numpy as np from lmfit import Model, Parameters from scipy.integrate import simps - import uncertainties as u +def bin_data(array, binsize): + if isinstance(binsize, int) and 0 < binsize < len(array): + return [np.mean(array[binsize * i:binsize * i + binsize]) for i in range(int( + np.ceil(len(array) / binsize)))] + else: + print("Binsize need to be positive integer smaller than lenght of array") + return array def find_nearest(array, value): # find nearest value and return index @@ -30,6 +36,7 @@ def fitccl( constraints_max, numfit_min=None, numfit_max=None, + binning=None ): """Made for fitting of ccl date where 1 peak is expected. Allows for combination of gaussian and linear model combination :param data: dictionary after peak fining @@ -40,6 +47,7 @@ def fitccl( :param numfit_max: maximal value on x axis for numerical integration - if none is centre of gaussian plus 3 sigma :param constraints_min: min constranits value for fit :param constraints_max: max constranits value for fit + :param binning : binning of the data :return data dict with additional values order for guess, vary, constraints_min, constraints_max: [Gaussian centre, Gaussian sigma, Gaussian amplitude, background slope, background intercept] @@ -56,10 +64,21 @@ def fitccl( print("More than 1 peak, measurement skipped") return - x = list(meas["om"]) - y = list(meas["Counts"]) - # if the dictionaries were merged/substracted, takes the errors from them, if not, takes them as sqrt(y) - y_err = np.sqrt(y) if meas.get("sigma", None) is None else meas.get("sigma") + if binning is None or binning == 0 or binning == 1: + x = list(meas["om"]) + y = list(meas["Counts"]) + y_err = list(np.sqrt(y)) if meas.get("sigma", None) is None else list(meas["sigma"]) + centre = x[int(meas["peak_indexes"])] + else: + x = list(meas["om"]) + centre = x[int(meas["peak_indexes"])] + x = bin_data(x, binning) + y = list(meas["Counts"]) + y_err = list(np.sqrt(y)) if meas.get("sigma", None) is None else list(meas["sigma"]) + combined = bin_data(create_uncertanities(y,y_err), binning) + y = [combined[i].n for i in range(len(combined))] + y_err = [combined[i].s for i in range(len(combined))] + if len(meas["peak_indexes"]) == 0: # Case for no peak, gaussian in centre, sigma as 20% of range @@ -75,8 +94,8 @@ def fitccl( elif len(meas["peak_indexes"]) == 1: # case for one peak, takse into account users guesses print("one peak") - peak_index, peak_height = meas["peak_indexes"], meas["peak_heights"] - guess[0] = x[int(peak_index)] if guess[0] is None else guess[0] + peak_height = meas["peak_heights"] + guess[0] = centre if guess[0] is None else guess[0] guess[1] = 0.1 if guess[1] is None else guess[1] guess[2] = float(peak_height / 10) if guess[2] is None else float(guess[2]) guess[3] = 0 if guess[3] is None else guess[3] @@ -84,7 +103,7 @@ def fitccl( constraints_min[0] = np.min(x) if constraints_min[0] is None else constraints_min[0] constraints_max[0] = np.max(x) if constraints_max[0] is None else constraints_max[0] - centre = x[int(peak_index)] + def gaussian(x, g_cen, g_width, g_amp): """1-d gaussian: gaussian(x, amp, cen, wid)""" From 0febab42924f63c4635c403e0de8f32e843fa633 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Tue, 13 Oct 2020 12:18:38 +0200 Subject: [PATCH 076/116] Modules for parametric study Loading files from txt and list of filenames, creating and sorting the dataframe and making of the graphs --- pyzebra/param_study_moduls.py | 146 ++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 pyzebra/param_study_moduls.py diff --git a/pyzebra/param_study_moduls.py b/pyzebra/param_study_moduls.py new file mode 100644 index 0000000..b2f3e2d --- /dev/null +++ b/pyzebra/param_study_moduls.py @@ -0,0 +1,146 @@ +from load_1Dnn import load_1D +from ccl_dict_operation import add_dict +import pandas as pd +from mpl_toolkits.mplot3d import Axes3D # dont delete, otherwise waterfall wont work +import matplotlib.pyplot as plt +import matplotlib as mpl +import numpy as np + + +def load_dats(filepath): + """reads the txt file, get headers and data + :arg filepath to txt file or list of filepaths to the files + :return ccl like dictionary""" + if isinstance(filepath, str): + data_type = "txt" + file_list = list() + with open(filepath, "r") as infile: + col_names = next(infile).split() + for line in infile: + if "END" in line: + break + file_list.append(tuple(line.split())) + elif isinstance(filepath, list): + data_type = "list" + file_list = filepath + + print(file_list) + dict1 = {} + for i in range(len(file_list)): + if not dict1: + if data_type == "txt": + dict1 = load_1D(file_list[0][0]) + else: + dict1 = load_1D(file_list[0]) + else: + if data_type == "txt": + dict1 = add_dict(dict1, load_1D(file_list[i][0])) + else: + dict1 = add_dict(dict1, load_1D(file_list[i])) + dict1["Measurements"][str("M" + str(i + 1))]["params"] = {} + if data_type == "txt": + for x in range(len(col_names) - 1): + dict1["Measurements"][str("M" + str(i + 1))]["params"][ + col_names[x + 1] + ] = file_list[i][x + 1] + + return dict1 + + +def create_dataframe(dict1): + """Creates pandas dataframe from the dictionary + :arg ccl like dictionary + :return pandas dataframe""" + # create dictionary to which we pull only wanted items before transforming it to pd.dataframe + pull_dict = {} + pull_dict["filenames"] = list() + for key in dict1["Measurements"]["M1"]["params"]: + pull_dict[key] = list() + pull_dict["temperature"] = list() + pull_dict["mag_field"] = list() + pull_dict["fit_area"] = list() + pull_dict["int_area"] = list() + pull_dict["om"] = list() + pull_dict["Counts"] = list() + + # populate the dict + for keys in dict1["Measurements"]: + if "file_of_origin" in dict1["Measurements"][keys]: + pull_dict["filenames"].append( + dict1["Measurements"][keys]["file_of_origin"].split("/")[-1] + ) + else: + pull_dict["filenames"].append(dict1["meta"]["original_filename"].split("/")[-1]) + for key in dict1["Measurements"][keys]["params"]: + pull_dict[str(key)].append(float(dict1["Measurements"][keys]["params"][key])) + pull_dict["temperature"].append(dict1["Measurements"][keys]["temperature"]) + pull_dict["mag_field"].append(dict1["Measurements"][keys]["mag_field"]) + pull_dict["fit_area"].append(dict1["Measurements"][keys]["fit"]["fit_area"]) + pull_dict["int_area"].append(dict1["Measurements"][keys]["fit"]["int_area"]) + pull_dict["om"].append(dict1["Measurements"][keys]["om"]) + pull_dict["Counts"].append(dict1["Measurements"][keys]["Counts"]) + + return pd.DataFrame(data=pull_dict) + + +def sort_dataframe(dataframe, sorting_parameter): + """sorts the data frame and resets index""" + data = dataframe.sort_values(by=sorting_parameter) + data = data.reset_index(drop=True) + return data + + +def make_graph(data, sorting_parameter, style): + """Makes the graph from the data based on style and sorting parameter + :arg data : pandas dataframe with data after sorting + :arg sorting_parameter to pull the correct variable and name + :arg style of the graph - waterfall, scatter, heatmap + :return matplotlib figure""" + if style == "waterfall": + mpl.rcParams["legend.fontsize"] = 10 + fig = plt.figure() + ax = fig.gca(projection="3d") + for i in range(len(data)): + x = data["om"][i] + z = data["Counts"][i] + yy = [data[sorting_parameter][i]] * len(x) + ax.plot(x, yy, z, label=str("%s = %f" % (sorting_parameter, yy[i]))) + + ax.legend() + ax.set_xlabel("Omega") + ax.set_ylabel(sorting_parameter) + ax.set_zlabel("counts") + + elif style == "scatter": + fig = plt.figure() + plt.errorbar( + data[sorting_parameter], + [data["fit_area"][i].n for i in range(len(data["fit_area"]))], + [data["fit_area"][i].s for i in range(len(data["fit_area"]))], + capsize=5, + ecolor="green", + ) + plt.xlabel(str(sorting_parameter)) + plt.ylabel("Intesity") + + elif style == "heat": + new_om = list() + for i in range(len(data)): + new_om = np.append(new_om, np.around(data["om"][i], 2), axis=0) + unique_om = np.unique(new_om) + color_matrix = np.zeros(shape=(len(data), len(unique_om))) + for i in range(len(data)): + for j in range(len(data["om"][i])): + if np.around(data["om"][i][j], 2) in np.unique(new_om): + color_matrix[i, j] = data["Counts"][i][j] + else: + continue + + fig = plt.figure() + plt.pcolormesh(unique_om, data[sorting_parameter], color_matrix, shading="gouraud") + plt.xlabel("omega") + plt.ylabel(sorting_parameter) + plt.colorbar() + plt.clim(color_matrix.mean(), color_matrix.max()) + + return fig From a25fab93caa2345feed8b96e5c8acbbda0492a66 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 13 Oct 2020 11:48:47 +0200 Subject: [PATCH 077/116] Optimize layout --- pyzebra/app/panel_ccl_integrate.py | 74 +++++++++++++++--------------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 97fb970..50e11e8 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -64,7 +64,7 @@ def create(): ccl_file_select.options = ccl_file_list ccl_file_select.value = ccl_file_list[0][0] - proposal_textinput = TextInput(title="Enter proposal number:") + proposal_textinput = TextInput(title="Enter proposal number:", default_size=145) proposal_textinput.on_change("value", proposal_textinput_callback) def ccl_file_select_callback(_attr, _old, new): @@ -224,14 +224,14 @@ def create(): _update_table() _update_plot(meas_name) - peak_pos_textinput = TextInput(title="Peak position:") + peak_pos_textinput = TextInput(title="Peak position:", default_size=145) peak_pos_textinput.on_change("value", peak_pos_textinput_callback) peak_int_ratio_spinner = Spinner( title="Peak intensity ratio:", value=0.8, step=0.01, low=0, high=1, default_size=145 ) peak_prominence_spinner = Spinner(title="Peak prominence:", value=50, low=0, default_size=145) - smooth_toggle = Toggle(label="Smooth curve") + smooth_toggle = Toggle(label="Smooth curve", default_size=145) window_size_spinner = Spinner(title="Window size:", value=7, step=2, low=1, default_size=145) poly_order_spinner = Spinner(title="Poly order:", value=3, low=0, default_size=145) @@ -282,7 +282,7 @@ def create(): integ_from.value = None integ_to.value = None - fitparam_reset_button = Button(label="Reset to defaults") + fitparam_reset_button = Button(label="Reset to defaults", default_size=145) fitparam_reset_button.on_click(fitparam_reset_button_callback) fit_output_textinput = TextAreaInput(title="Fit results:", width=450, height=400) @@ -305,7 +305,7 @@ def create(): sel_ind = meas_table_source.selected.indices[-1] _update_plot(meas_table_source.data["measurement"][sel_ind]) - peakfind_all_button = Button(label="Peak Find All", button_type="primary") + peakfind_all_button = Button(label="Peak Find All", button_type="primary", default_size=145) peakfind_all_button.on_click(peakfind_all_button_callback) def peakfind_button_callback(): @@ -325,7 +325,7 @@ def create(): _update_table() _update_plot(meas) - peakfind_button = Button(label="Peak Find Current") + peakfind_button = Button(label="Peak Find Current", default_size=145) peakfind_button.on_click(peakfind_button_callback) def fit_all_button_callback(): @@ -371,7 +371,7 @@ def create(): sel_ind = meas_table_source.selected.indices[-1] _update_plot(meas_table_source.data["measurement"][sel_ind]) - fit_all_button = Button(label="Fit All", button_type="primary") + fit_all_button = Button(label="Fit All", button_type="primary", default_size=145) fit_all_button.on_click(fit_all_button_callback) def fit_button_callback(): @@ -416,7 +416,7 @@ def create(): _update_plot(meas) - fit_button = Button(label="Fit Current") + fit_button = Button(label="Fit Current", default_size=145) fit_button.on_click(fit_button_callback) def export_fit_checkbox_callback(_attr, _old, new): @@ -469,45 +469,43 @@ def create(): save_button.js_on_click(CustomJS(args={"js_data": js_data}, code=javaScript)) findpeak_controls = column( - peak_pos_textinput, + row(peak_pos_textinput, column(Spacer(height=19), smooth_toggle)), row(peak_int_ratio_spinner, peak_prominence_spinner), - smooth_toggle, row(window_size_spinner, poly_order_spinner), - peakfind_button, - peakfind_all_button, + row(peakfind_button, peakfind_all_button), ) div_1 = Div(text="Guess:") div_2 = Div(text="Vary:") div_3 = Div(text="Min:") div_4 = Div(text="Max:") - div_5 = Div(text="Gauss Centre:") - div_6 = Div(text="Gauss Sigma:") - div_7 = Div(text="Gauss Ampl.:") - div_8 = Div(text="Slope:") - div_9 = Div(text="Offset:") - fitpeak_controls = column( - row( - column( - Spacer(height=36), - div_1, - Spacer(height=12), - div_2, - Spacer(height=12), - div_3, - Spacer(height=12), - div_4, - ), - column(div_5, centre_guess, centre_vary, centre_min, centre_max), - column(div_6, sigma_guess, sigma_vary, sigma_min, sigma_max), - column(div_7, ampl_guess, ampl_vary, ampl_min, ampl_max), - column(div_8, slope_guess, slope_vary, slope_min, slope_max), - column(div_9, offset_guess, offset_vary, offset_min, offset_max), + div_5 = Div(text="Gauss Centre:", margin=[5, 5, -5, 5]) + div_6 = Div(text="Gauss Sigma:", margin=[5, 5, -5, 5]) + div_7 = Div(text="Gauss Ampl.:", margin=[5, 5, -5, 5]) + div_8 = Div(text="Slope:", margin=[5, 5, -5, 5]) + div_9 = Div(text="Offset:", margin=[5, 5, -5, 5]) + fitpeak_controls = row( + column( + Spacer(height=36), + div_1, + Spacer(height=12), + div_2, + Spacer(height=12), + div_3, + Spacer(height=12), + div_4, + ), + column(div_5, centre_guess, centre_vary, centre_min, centre_max), + column(div_6, sigma_guess, sigma_vary, sigma_min, sigma_max), + column(div_7, ampl_guess, ampl_vary, ampl_min, ampl_max), + column(div_8, slope_guess, slope_vary, slope_min, slope_max), + column(div_9, offset_guess, offset_vary, offset_min, offset_max), + Spacer(width=20), + column( + row(integ_from, integ_to), + row(fitparam_reset_button, column(Spacer(height=5), export_fit_checkbox)), + row(fit_button, fit_all_button), ), - row(integ_from, integ_to, column(Spacer(height=25), export_fit_checkbox)), - row(fitparam_reset_button), - row(fit_button), - row(fit_all_button), ) export_layout = column(preview_output_textinput, row(preview_output_button, save_button)) From 6edc9c2ec48dbb604f486c2433b314c755585fc6 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Tue, 13 Oct 2020 17:50:26 +0200 Subject: [PATCH 078/116] Update param_study_moduls.py changed the delimiter to "," --- pyzebra/param_study_moduls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyzebra/param_study_moduls.py b/pyzebra/param_study_moduls.py index b2f3e2d..2c7618e 100644 --- a/pyzebra/param_study_moduls.py +++ b/pyzebra/param_study_moduls.py @@ -15,11 +15,11 @@ def load_dats(filepath): data_type = "txt" file_list = list() with open(filepath, "r") as infile: - col_names = next(infile).split() + col_names = next(infile).split(",") for line in infile: if "END" in line: break - file_list.append(tuple(line.split())) + file_list.append(tuple(line.split(","))) elif isinstance(filepath, list): data_type = "list" file_list = filepath From fc70bdcb1386ccbd601f35c4400b706525f25d9c Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Tue, 13 Oct 2020 18:07:47 +0200 Subject: [PATCH 079/116] Update fit2.py fixed the weighted fit --- pyzebra/fit2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 6d68a09..fdec9b1 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -157,7 +157,7 @@ def fitccl( ), ) # the weighted fit - result = mod.fit(y, params, weights=y_err, x=x, calc_covar=True) + result = mod.fit(y, params, weights=[1/y_err[i] for i in range(len(y_err))], x=x, calc_covar=True) # u.ufloat to work with uncertanities fit_area = u.ufloat(result.params["g_amp"].value, result.params["g_amp"].stderr) comps = result.eval_components() From 6886279884ec7a70019949eb830b59468e08ebb2 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Tue, 13 Oct 2020 18:15:20 +0200 Subject: [PATCH 080/116] Update fit2.py added abs value to weights just in case --- pyzebra/fit2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index fdec9b1..edfa49c 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -157,7 +157,7 @@ def fitccl( ), ) # the weighted fit - result = mod.fit(y, params, weights=[1/y_err[i] for i in range(len(y_err))], x=x, calc_covar=True) + result = mod.fit(y, params, weights=[np.abs(1/y_err[i]) for i in range(len(y_err))], x=x, calc_covar=True) # u.ufloat to work with uncertanities fit_area = u.ufloat(result.params["g_amp"].value, result.params["g_amp"].stderr) comps = result.eval_components() From 0777d735663bd339a5a2a499455aa115062c23d2 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Wed, 14 Oct 2020 15:32:38 +0200 Subject: [PATCH 081/116] Cleanup, import sort, formatting --- pyzebra/__init__.py | 14 +++---- pyzebra/app/save_load_dict.py | 19 --------- pyzebra/ccl_dict_operation.py | 1 + pyzebra/fit2.py | 68 ++++++++++---------------------- pyzebra/h5.py | 19 +-------- pyzebra/load_1D.py | 4 +- pyzebra/param_study_moduls.py | 11 +++--- pyzebra/visualize peak fitter.py | 39 ------------------ 8 files changed, 37 insertions(+), 138 deletions(-) delete mode 100644 pyzebra/app/save_load_dict.py delete mode 100644 pyzebra/visualize peak fitter.py diff --git a/pyzebra/__init__.py b/pyzebra/__init__.py index 847d3fa..db37dcf 100644 --- a/pyzebra/__init__.py +++ b/pyzebra/__init__.py @@ -1,8 +1,8 @@ -from pyzebra.anatric import * -from pyzebra.h5 import * -from pyzebra.xtal import * -from pyzebra.load_1D import load_1D, parse_1D -from pyzebra.ccl_findpeaks import ccl_findpeaks -from pyzebra.fit2 import fitccl -from pyzebra.comm_export import export_comm import pyzebra.ccl_dict_operation +from pyzebra.anatric import * +from pyzebra.ccl_findpeaks import ccl_findpeaks +from pyzebra.comm_export import export_comm +from pyzebra.fit2 import fitccl +from pyzebra.h5 import * +from pyzebra.load_1D import load_1D, parse_1D +from pyzebra.xtal import * diff --git a/pyzebra/app/save_load_dict.py b/pyzebra/app/save_load_dict.py deleted file mode 100644 index 5e31442..0000000 --- a/pyzebra/app/save_load_dict.py +++ /dev/null @@ -1,19 +0,0 @@ -import pickle - - -def save_dict(obj, name): - """ saves dictionary as pickle file in binary format - :arg obj - object to save - :arg name - name of the file - NOTE: path should be added later""" - with open(name + '.pkl', 'wb') as f: - pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL) - - -def load_dict(name): - """load dictionary from picle file - :arg name - name of the file to load - NOTE: expect the file in the same folder, path should be added later - :return dictionary""" - with open(name + '.pkl', 'rb') as f: - return pickle.load(f) diff --git a/pyzebra/ccl_dict_operation.py b/pyzebra/ccl_dict_operation.py index 755a4af..48fc88e 100644 --- a/pyzebra/ccl_dict_operation.py +++ b/pyzebra/ccl_dict_operation.py @@ -1,5 +1,6 @@ import numpy as np import uncertainties as u + from .fit2 import create_uncertanities diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index edfa49c..0c67c42 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -1,16 +1,20 @@ import numpy as np +import uncertainties as u from lmfit import Model, Parameters from scipy.integrate import simps -import uncertainties as u + def bin_data(array, binsize): if isinstance(binsize, int) and 0 < binsize < len(array): - return [np.mean(array[binsize * i:binsize * i + binsize]) for i in range(int( - np.ceil(len(array) / binsize)))] + return [ + np.mean(array[binsize * i : binsize * i + binsize]) + for i in range(int(np.ceil(len(array) / binsize))) + ] else: print("Binsize need to be positive integer smaller than lenght of array") return array + def find_nearest(array, value): # find nearest value and return index array = np.asarray(array) @@ -36,7 +40,7 @@ def fitccl( constraints_max, numfit_min=None, numfit_max=None, - binning=None + binning=None, ): """Made for fitting of ccl date where 1 peak is expected. Allows for combination of gaussian and linear model combination :param data: dictionary after peak fining @@ -75,17 +79,16 @@ def fitccl( x = bin_data(x, binning) y = list(meas["Counts"]) y_err = list(np.sqrt(y)) if meas.get("sigma", None) is None else list(meas["sigma"]) - combined = bin_data(create_uncertanities(y,y_err), binning) + combined = bin_data(create_uncertanities(y, y_err), binning) y = [combined[i].n for i in range(len(combined))] y_err = [combined[i].s for i in range(len(combined))] - if len(meas["peak_indexes"]) == 0: # Case for no peak, gaussian in centre, sigma as 20% of range print("No peak") peak_index = find_nearest(x, np.mean(x)) guess[0] = x[int(peak_index)] if guess[0] is None else guess[0] - guess[1] = (x[-1] - x[0])/5 if guess[1] is None else guess[1] + guess[1] = (x[-1] - x[0]) / 5 if guess[1] is None else guess[1] guess[2] = 50 if guess[2] is None else guess[2] guess[3] = 0 if guess[3] is None else guess[3] guess[4] = np.mean(y) if guess[4] is None else guess[4] @@ -103,8 +106,6 @@ def fitccl( constraints_min[0] = np.min(x) if constraints_min[0] is None else constraints_min[0] constraints_max[0] = np.max(x) if constraints_max[0] is None else constraints_max[0] - - def gaussian(x, g_cen, g_width, g_amp): """1-d gaussian: gaussian(x, amp, cen, wid)""" return (g_amp / (np.sqrt(2 * np.pi) * g_width)) * np.exp( @@ -119,49 +120,19 @@ def fitccl( params = Parameters() params.add_many( ("g_cen", x[int(peak_index)], bool(vary[0]), np.min(x), np.max(x), None, None), - ( - "g_width", - guess[1], - bool(vary[1]), - constraints_min[1], - constraints_max[1], - None, - None, - ), - ( - "g_amp", - guess[2], - bool(vary[2]), - constraints_min[2], - constraints_max[2], - None, - None, - ), - ( - "slope", - guess[3], - bool(vary[3]), - constraints_min[3], - constraints_max[3], - None, - None, - ), - ( - "intercept", - guess[4], - bool(vary[4]), - constraints_min[4], - constraints_max[4], - None, - None, - ), + ("g_width", guess[1], bool(vary[1]), constraints_min[1], constraints_max[1], None, None,), + ("g_amp", guess[2], bool(vary[2]), constraints_min[2], constraints_max[2], None, None,), + ("slope", guess[3], bool(vary[3]), constraints_min[3], constraints_max[3], None, None,), + ("intercept", guess[4], bool(vary[4]), constraints_min[4], constraints_max[4], None, None,), ) # the weighted fit - result = mod.fit(y, params, weights=[np.abs(1/y_err[i]) for i in range(len(y_err))], x=x, calc_covar=True) + result = mod.fit( + y, params, weights=[np.abs(1 / y_err[i]) for i in range(len(y_err))], x=x, calc_covar=True + ) # u.ufloat to work with uncertanities fit_area = u.ufloat(result.params["g_amp"].value, result.params["g_amp"].stderr) comps = result.eval_components() - + if result.params["g_amp"].stderr is None: result.params["g_amp"].stderr = result.params["g_amp"].value elif result.params["g_amp"].stderr > result.params["g_amp"].value: @@ -183,7 +154,8 @@ def fitccl( it = -1 while abs(numfit_max - numfit_min) < 3: - # in the case the peak is very thin and numerical integration would be on zero omega difference, finds closes values + # in the case the peak is very thin and numerical integration would be on zero omega + # difference, finds closes values it = it + 1 numfit_min = find_nearest( x, diff --git a/pyzebra/h5.py b/pyzebra/h5.py index 14f2e4b..3286f0e 100644 --- a/pyzebra/h5.py +++ b/pyzebra/h5.py @@ -22,7 +22,7 @@ def parse_h5meta(file): for line in file: line = line.strip() if line.startswith("#begin "): - section = line[len("#begin "):] + section = line[len("#begin ") :] content[section] = [] elif line.startswith("#end"): @@ -64,20 +64,3 @@ def read_detector_data(filepath): det_data["temperature"] = h5f["/entry1/sample/temperature"][:] return det_data - - -def open_h5meta(filepath): - """Open h5meta file like *.cami - - Args: - filepath (str): File path of a h5meta file. - - Returns: - dict: A dictionary with h5 names and their detector data and angles. - """ - data = dict() - h5meta_content = read_h5meta(filepath) - for file in h5meta_content["filelist"]: - data[file] = read_detector_data(file) - - return data diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 2d83a41..a2fbed9 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -97,7 +97,7 @@ def parse_1D(fileobj, data_type): # read data if data_type == ".ccl": - metadata['data_type'] = data_type + metadata["data_type"] = data_type measurements = {} decimal = list() data = fileobj.readlines() @@ -162,7 +162,7 @@ def parse_1D(fileobj, data_type): elif data_type == ".dat": # skip the first 2 rows, the third row contans the column names - metadata['data_type'] = data_type + metadata["data_type"] = data_type next(fileobj) next(fileobj) col_names = next(fileobj).split() diff --git a/pyzebra/param_study_moduls.py b/pyzebra/param_study_moduls.py index 2c7618e..e04afe1 100644 --- a/pyzebra/param_study_moduls.py +++ b/pyzebra/param_study_moduls.py @@ -1,10 +1,11 @@ -from load_1Dnn import load_1D -from ccl_dict_operation import add_dict -import pandas as pd -from mpl_toolkits.mplot3d import Axes3D # dont delete, otherwise waterfall wont work -import matplotlib.pyplot as plt import matplotlib as mpl +import matplotlib.pyplot as plt import numpy as np +import pandas as pd +from load_1Dnn import load_1D +from mpl_toolkits.mplot3d import Axes3D # dont delete, otherwise waterfall wont work + +from ccl_dict_operation import add_dict def load_dats(filepath): diff --git a/pyzebra/visualize peak fitter.py b/pyzebra/visualize peak fitter.py deleted file mode 100644 index fc4b034..0000000 --- a/pyzebra/visualize peak fitter.py +++ /dev/null @@ -1,39 +0,0 @@ -plt.figure(figsize=(20, 10)) -plt.plot(x, y, "b-", label="Original data") -plt.plot(x, comps["gaussian"], "r--", label="Gaussian component") -plt.fill_between(x, comps["gaussian"], facecolor="red", alpha=0.4) -plt.plot(x, comps["background"], "g--", label="Line component") -plt.fill_between(x, comps["background"], facecolor="green", alpha=0.4) - -plt.fill_between( - x[numfit_min:numfit_max], - y[numfit_min:numfit_max], - facecolor="yellow", - alpha=0.4, - label="Integrated area", ) - -plt.plot(x, result.best_fit, "k-", label="Best fit") -plt.title( - "%s \n Gaussian: centre = %9.4f, sigma = %9.4f, area = %9.4f \n" - "background: slope = %9.4f, intercept = %9.4f \n" - "Int. area = %9.4f +/- %9.4f \n" - "fit area = %9.4f +/- %9.4f \n" - "ratio((fit-int)/fit) = %9.4f" - % ( - keys, - result.params["g_cen"].value, - result.params["g_width"].value, - result.params["g_amp"].value, - result.params["slope"].value, - result.params["intercept"].value, - int_area.n, - int_area.s, - result.params["g_amp"].value, - result.params["g_amp"].stderr, - (result.params["g_amp"].value - int_area.n) / result.params["g_amp"].value - - ) -) - -plt.legend(loc="best") -plt.show() \ No newline at end of file From 8bf62274c02da45d085cbb24785333c36872b839 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Wed, 14 Oct 2020 16:31:20 +0200 Subject: [PATCH 082/116] Update fit2.py --- pyzebra/fit2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 0c67c42..dbb7179 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -87,7 +87,7 @@ def fitccl( # Case for no peak, gaussian in centre, sigma as 20% of range print("No peak") peak_index = find_nearest(x, np.mean(x)) - guess[0] = x[int(peak_index)] if guess[0] is None else guess[0] + guess[0] = centre if guess[0] is None else guess[0] guess[1] = (x[-1] - x[0]) / 5 if guess[1] is None else guess[1] guess[2] = 50 if guess[2] is None else guess[2] guess[3] = 0 if guess[3] is None else guess[3] @@ -119,7 +119,7 @@ def fitccl( mod = Model(gaussian) + Model(background) params = Parameters() params.add_many( - ("g_cen", x[int(peak_index)], bool(vary[0]), np.min(x), np.max(x), None, None), + ("g_cen", guess[0], bool(vary[0]), np.min(x), np.max(x), None, None), ("g_width", guess[1], bool(vary[1]), constraints_min[1], constraints_max[1], None, None,), ("g_amp", guess[2], bool(vary[2]), constraints_min[2], constraints_max[2], None, None,), ("slope", guess[3], bool(vary[3]), constraints_min[3], constraints_max[3], None, None,), From 2e737df2bfeec7b6df966165eebc1dc5db6ebd78 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Wed, 14 Oct 2020 16:43:33 +0200 Subject: [PATCH 083/116] Update fit2.py Second apology, that broke the code in another way, hopefully now its alrigh --- pyzebra/fit2.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index dbb7179..ff8166d 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -67,19 +67,25 @@ def fitccl( # return in case of more than 1 peaks print("More than 1 peak, measurement skipped") return - if binning is None or binning == 0 or binning == 1: x = list(meas["om"]) y = list(meas["Counts"]) y_err = list(np.sqrt(y)) if meas.get("sigma", None) is None else list(meas["sigma"]) - centre = x[int(meas["peak_indexes"])] + print(meas["peak_indexes"]) + if not meas["peak_indexes"]: + centre = np.mean(x) + else: + centre = x[int(meas["peak_indexes"])] else: x = list(meas["om"]) - centre = x[int(meas["peak_indexes"])] + if not meas["peak_indexes"]: + centre = np.mean(x) + else: + centre = x[int(meas["peak_indexes"])] x = bin_data(x, binning) y = list(meas["Counts"]) y_err = list(np.sqrt(y)) if meas.get("sigma", None) is None else list(meas["sigma"]) - combined = bin_data(create_uncertanities(y, y_err), binning) + combined = bin_data(create_uncertanities(y,y_err), binning) y = [combined[i].n for i in range(len(combined))] y_err = [combined[i].s for i in range(len(combined))] From 29c3b647a988f547de0529d0679be050e2c6f8de Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Wed, 14 Oct 2020 18:04:19 +0200 Subject: [PATCH 084/116] Add basic display for stdout and bokeh logger --- pyzebra/app/app.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/pyzebra/app/app.py b/pyzebra/app/app.py index 67f4e14..35e5bf0 100644 --- a/pyzebra/app/app.py +++ b/pyzebra/app/app.py @@ -1,11 +1,15 @@ import argparse +import logging +import sys +from io import StringIO from bokeh.io import curdoc -from bokeh.models import Tabs +from bokeh.layouts import column, row +from bokeh.models import Tabs, TextAreaInput +import panel_ccl_integrate import panel_hdf_anatric import panel_hdf_viewer -import panel_ccl_integrate parser = argparse.ArgumentParser( prog="pyzebra", formatter_class=argparse.ArgumentDefaultsHelpFormatter @@ -16,9 +20,32 @@ args = parser.parse_args() doc = curdoc() doc.title = "pyzebra" +sys.stdout = StringIO() +stdout_textareainput = TextAreaInput(title="print output:", height=150) + +bokeh_stream = StringIO() +bokeh_handler = logging.StreamHandler(bokeh_stream) +bokeh_handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) +bokeh_logger = logging.getLogger('bokeh') +bokeh_logger.addHandler(bokeh_handler) +bokeh_log_textareainput = TextAreaInput(title="server output:", height=150) + # Final layout tab_hdf_viewer = panel_hdf_viewer.create() tab_hdf_anatric = panel_hdf_anatric.create() tab_ccl_integrate = panel_ccl_integrate.create() -doc.add_root(Tabs(tabs=[tab_hdf_viewer, tab_hdf_anatric, tab_ccl_integrate])) +doc.add_root( + column( + Tabs(tabs=[tab_hdf_viewer, tab_hdf_anatric, tab_ccl_integrate]), + row(stdout_textareainput, bokeh_log_textareainput, sizing_mode="scale_both"), + ) +) + + +def update_stdout(): + stdout_textareainput.value = sys.stdout.getvalue() + bokeh_log_textareainput.value = bokeh_stream.getvalue() + + +doc.add_periodic_callback(update_stdout, 1000) From c04e6fbf1b9232938b5a92ff4d781b4d31ea867f Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 15 Oct 2020 13:48:03 +0200 Subject: [PATCH 085/116] Replace export_fit checkbox with radiobutton group The area method is now applying for all measurements --- pyzebra/app/panel_ccl_integrate.py | 21 ++++++++------------- pyzebra/comm_export.py | 4 ++-- pyzebra/fit2.py | 3 +-- pyzebra/load_1D.py | 6 ++++-- 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 50e11e8..8522e27 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -7,7 +7,6 @@ from bokeh.layouts import column, row from bokeh.models import ( BasicTicker, Button, - CheckboxGroup, Circle, ColumnDataSource, CustomJS, @@ -20,6 +19,7 @@ from bokeh.models import ( LinearAxis, Panel, Plot, + RadioButtonGroup, Select, Spacer, Spinner, @@ -128,11 +128,6 @@ def create(): fit = meas.get("fit") if fit is not None: - if meas["fit"]["export_fit"]: - export_fit_checkbox.active = [0] - else: - export_fit_checkbox.active = [] - plot_gauss_source.data.update(x=x, y=meas["fit"]["comps"]["gaussian"]) plot_bkg_source.data.update(x=x, y=meas["fit"]["comps"]["background"]) params = fit["result"].params @@ -419,13 +414,13 @@ def create(): fit_button = Button(label="Fit Current", default_size=145) fit_button.on_click(fit_button_callback) - def export_fit_checkbox_callback(_attr, _old, new): - sel_ind = meas_table_source.selected.indices[-1] - meas = meas_table_source.data["measurement"][sel_ind] - det_data["Measurements"][meas]["fit"]["export_fit"] = bool(new) + def area_method_radiobutton_callback(_attr, _old, new): + det_data["meta"]["area_method"] = ("fit", "integ")[new] - export_fit_checkbox = CheckboxGroup(labels=["Export fit"], width=100) - export_fit_checkbox.on_change("active", export_fit_checkbox_callback) + area_method_radiobutton = RadioButtonGroup( + labels=["Fit", "Integral"], active=0, default_size=145 + ) + area_method_radiobutton.on_change("active", area_method_radiobutton_callback) preview_output_textinput = TextAreaInput(title="Export file preview:", width=450, height=400) @@ -503,7 +498,7 @@ def create(): Spacer(width=20), column( row(integ_from, integ_to), - row(fitparam_reset_button, column(Spacer(height=5), export_fit_checkbox)), + row(fitparam_reset_button, area_method_radiobutton), row(fit_button, fit_all_button), ), ) diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py index 8211dac..cb70126 100644 --- a/pyzebra/comm_export.py +++ b/pyzebra/comm_export.py @@ -38,12 +38,12 @@ def export_comm(data, path, lorentz=False): h_str = f'{int(meas["h_index"]):{padding[1]}}' k_str = f'{int(meas["k_index"]):{padding[1]}}' l_str = f'{int(meas["l_index"]):{padding[1]}}' - if meas["fit"]["export_fit"] is True: + if data["meta"]["area_method"] == "fit": area = float(meas["fit"]["fit_area"].n) sigma_str = ( f'{"{:8.2f}".format(float(meas["fit"]["fit_area"].s)):{align}{padding[2]}}' ) - else: + elif data["meta"]["area_method"] == "integ": area = float(meas["fit"]["int_area"].n) sigma_str = ( f'{"{:8.2f}".format(float(meas["fit"]["int_area"].s)):{align}{padding[2]}}' diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index ff8166d..0b92550 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -216,8 +216,7 @@ def fitccl( print(result.fit_report()) print((result.params["g_amp"].value - int_area.n) / result.params["g_amp"].value) - d["export_fit"] = False - # ["export_fit"] = False if user wants num. int. value in comm/incomm, otherwise true + d["ratio"] = (result.params["g_amp"].value - int_area.n) / result.params["g_amp"].value d["int_area"] = int_area d["fit_area"] = u.ufloat(result.params["g_amp"].value, result.params["g_amp"].stderr) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index a2fbed9..7bac265 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -97,7 +97,6 @@ def parse_1D(fileobj, data_type): # read data if data_type == ".ccl": - metadata["data_type"] = data_type measurements = {} decimal = list() data = fileobj.readlines() @@ -162,7 +161,6 @@ def parse_1D(fileobj, data_type): elif data_type == ".dat": # skip the first 2 rows, the third row contans the column names - metadata["data_type"] = data_type next(fileobj) next(fileobj) col_names = next(fileobj).split() @@ -181,4 +179,8 @@ def parse_1D(fileobj, data_type): else: print("Unknown file extention") + # utility information + metadata["data_type"] = data_type + metadata["area_method"] = "fit" + return {"meta": metadata, "Measurements": measurements} From c5b0b17cb2998ef7554bb6bee431a1556bb83915 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 16 Oct 2020 10:33:46 +0200 Subject: [PATCH 086/116] Compact hdf viewer panel --- pyzebra/app/panel_hdf_viewer.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pyzebra/app/panel_hdf_viewer.py b/pyzebra/app/panel_hdf_viewer.py index ec1b183..c0e1409 100644 --- a/pyzebra/app/panel_hdf_viewer.py +++ b/pyzebra/app/panel_hdf_viewer.py @@ -376,7 +376,7 @@ def create(): overview_plot_x_image_glyph.color_mapper = LinearColorMapper(palette=cmap_dict[new]) overview_plot_y_image_glyph.color_mapper = LinearColorMapper(palette=cmap_dict[new]) - colormap = Select(title="Colormap:", options=list(cmap_dict.keys())) + colormap = Select(title="Colormap:", options=list(cmap_dict.keys()), default_size=145) colormap.on_change("value", colormap_callback) colormap.value = "plasma" @@ -395,7 +395,7 @@ def create(): update_image() - auto_toggle = Toggle(label="Auto Range", active=True, button_type="default") + auto_toggle = Toggle(label="Auto Range", active=True, button_type="default", default_size=145) auto_toggle.on_click(auto_toggle_callback) # ---- colormap display max value @@ -409,6 +409,7 @@ def create(): value=1, step=STEP, disabled=auto_toggle.active, + default_size=145, ) display_max_spinner.on_change("value", display_max_spinner_callback) @@ -423,6 +424,7 @@ def create(): value=0, step=STEP, disabled=auto_toggle.active, + default_size=145, ) display_min_spinner.on_change("value", display_min_spinner_callback) @@ -465,13 +467,22 @@ def create(): temperature_spinner = Spinner(title="Temperature:", format="0.00", width=145, disabled=True) # Final layout - layout_image = column( - gridplot([[proj_v, None], [plot, proj_h]], merge_tools=False), row(index_spinner) + layout_image = column(gridplot([[proj_v, None], [plot, proj_h]], merge_tools=False)) + colormap_layout = column( + row(colormap, column(Spacer(height=19), auto_toggle)), + row(display_max_spinner, display_min_spinner), ) - colormap_layout = column(colormap, auto_toggle, display_max_spinner, display_min_spinner) hkl_layout = column(radio_button_group, hkl_button) params_layout = row(magnetic_field_spinner, temperature_spinner) + layout_controls = row( + column(selection_button, selection_list), + Spacer(width=20), + column(frame_button_group, colormap_layout), + Spacer(width=20), + column(index_spinner, params_layout, hkl_layout), + ) + layout_overview = column( gridplot( [[overview_plot_x, overview_plot_y]], @@ -486,9 +497,9 @@ def create(): column( row(column(Spacer(height=5), upload_div), upload_button, filelist), layout_overview, - row(frame_button_group, selection_button, selection_list), + layout_controls, ), - column(roi_avg_plot, layout_image, row(colormap_layout, column(params_layout, hkl_layout))), + column(roi_avg_plot, layout_image), ) return Panel(child=tab_layout, title="hdf viewer") From cf8f19f6d95779ac7496eea58658d270e02a5fbe Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 16 Oct 2020 11:25:59 +0200 Subject: [PATCH 087/116] Use omega instead of a range sequence --- pyzebra/app/panel_ccl_integrate.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 8522e27..92b1672 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -3,6 +3,7 @@ import io import os import tempfile +import numpy as np from bokeh.layouts import column, row from bokeh.models import ( BasicTicker, @@ -105,7 +106,7 @@ def create(): meas = det_data["Measurements"][ind] y = meas["Counts"] - x = list(range(len(y))) + x = meas["om"] plot_line_source.data.update(x=x, y=y) @@ -113,11 +114,11 @@ def create(): if num_of_peaks is not None and num_of_peaks > 0: peak_indexes = meas["peak_indexes"] if len(peak_indexes) == 1: - peak_pos_textinput.value = str(peak_indexes[0]) + peak_pos_textinput.value = str(meas["om"][peak_indexes[0]]) else: - peak_pos_textinput.value = str(peak_indexes) + peak_pos_textinput.value = str([meas["om"][ind] for ind in peak_indexes]) - plot_circle_source.data.update(x=peak_indexes, y=meas["peak_heights"]) + plot_circle_source.data.update(x=meas["om"][peak_indexes], y=meas["peak_heights"]) plot_line_smooth_source.data.update(x=x, y=meas["smooth_peaks"]) else: peak_pos_textinput.value = None @@ -167,7 +168,7 @@ def create(): ) plot.add_layout(LinearAxis(axis_label="Counts"), place="left") - plot.add_layout(LinearAxis(), place="below") + plot.add_layout(LinearAxis(axis_label="Omega"), place="below") plot.add_layout(Grid(dimension=0, ticker=BasicTicker())) plot.add_layout(Grid(dimension=1, ticker=BasicTicker())) @@ -214,7 +215,8 @@ def create(): meas = det_data["Measurements"][meas_name] meas["num_of_peaks"] = 1 - meas["peak_indexes"] = [int(new)] + meas["peak_indexes"] = [(np.abs(meas["om"] - float(new))).argmin()] + print(meas["peak_indexes"]) meas["peak_heights"] = [0] _update_table() _update_plot(meas_name) From db4aabe89277f7d4e308a2ee98c0a98f0d642cb4 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 16 Oct 2020 11:53:27 +0200 Subject: [PATCH 088/116] Add vertical spans for integration borders --- pyzebra/app/panel_ccl_integrate.py | 11 +++++++++++ pyzebra/fit2.py | 1 + 2 files changed, 12 insertions(+) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 92b1672..8e8f4cf 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -23,6 +23,7 @@ from bokeh.models import ( RadioButtonGroup, Select, Spacer, + Span, Spinner, TableColumn, TextAreaInput, @@ -153,10 +154,14 @@ def create(): (params["g_amp"].value - fit["int_area"].n) / params["g_amp"].value, ) ) + numfit_min_span.location = x[fit["numfit"][0]] + numfit_max_span.location = x[fit["numfit"][1]] else: plot_gauss_source.data.update(x=[], y=[]) plot_bkg_source.data.update(x=[], y=[]) fit_output_textinput.value = "" + numfit_min_span.location = None + numfit_max_span.location = None # Main plot plot = Plot( @@ -190,6 +195,12 @@ def create(): plot_circle_source = ColumnDataSource(dict(x=[], y=[])) plot.add_glyph(plot_circle_source, Circle(x="x", y="y")) + numfit_min_span = Span(location=None, dimension='height', line_dash='dashed') + plot.add_layout(numfit_min_span) + + numfit_max_span = Span(location=None, dimension='height', line_dash='dashed') + plot.add_layout(numfit_max_span) + # Measurement select def meas_table_callback(_attr, _old, new): if new: diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 0b92550..a106b83 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -223,6 +223,7 @@ def fitccl( d["full_report"] = result.fit_report() d["result"] = result d["comps"] = comps + d["numfit"] = [numfit_min, numfit_max] meas["fit"] = d return data From de0c1b1f4b17baf08fa0c3aee124858b86969e17 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 16 Oct 2020 14:25:04 +0200 Subject: [PATCH 089/116] Fix manual peak_index entry --- pyzebra/app/panel_ccl_integrate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 8e8f4cf..065102a 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -226,9 +226,9 @@ def create(): meas = det_data["Measurements"][meas_name] meas["num_of_peaks"] = 1 - meas["peak_indexes"] = [(np.abs(meas["om"] - float(new))).argmin()] - print(meas["peak_indexes"]) - meas["peak_heights"] = [0] + peak_ind = (np.abs(meas["om"] - float(new))).argmin() + meas["peak_indexes"] = np.array([peak_ind], dtype=np.int64) + meas["peak_heights"] = np.array([meas["smooth_peaks"][peak_ind]]) _update_table() _update_plot(meas_name) From 306958b02a2d5a2eb7b1135469522099ca2d60ef Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 16 Oct 2020 14:46:19 +0200 Subject: [PATCH 090/116] Refactor handling of the result dict --- pyzebra/app/panel_ccl_integrate.py | 30 +++++++++++------------------- pyzebra/ccl_findpeaks.py | 27 ++++++++++----------------- pyzebra/fit2.py | 20 +++++++------------- 3 files changed, 28 insertions(+), 49 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 065102a..7587a24 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -195,10 +195,10 @@ def create(): plot_circle_source = ColumnDataSource(dict(x=[], y=[])) plot.add_glyph(plot_circle_source, Circle(x="x", y="y")) - numfit_min_span = Span(location=None, dimension='height', line_dash='dashed') + numfit_min_span = Span(location=None, dimension="height", line_dash="dashed") plot.add_layout(numfit_min_span) - numfit_max_span = Span(location=None, dimension='height', line_dash='dashed') + numfit_max_span = Span(location=None, dimension="height", line_dash="dashed") plot.add_layout(numfit_max_span) # Measurement select @@ -296,10 +296,8 @@ def create(): fit_output_textinput = TextAreaInput(title="Fit results:", width=450, height=400) def peakfind_all_button_callback(): - nonlocal det_data - for meas in det_data["Measurements"]: - det_data = pyzebra.ccl_findpeaks( - det_data, + for meas in det_data["Measurements"].values(): + pyzebra.ccl_findpeaks( meas, int_threshold=peak_int_ratio_spinner.value, prominence=peak_prominence_spinner.value, @@ -317,12 +315,10 @@ def create(): peakfind_all_button.on_click(peakfind_all_button_callback) def peakfind_button_callback(): - nonlocal det_data sel_ind = meas_table_source.selected.indices[-1] meas = meas_table_source.data["measurement"][sel_ind] - det_data = pyzebra.ccl_findpeaks( - det_data, - meas, + pyzebra.ccl_findpeaks( + det_data["Measurements"][meas], int_threshold=peak_int_ratio_spinner.value, prominence=peak_prominence_spinner.value, smooth=smooth_toggle.active, @@ -337,12 +333,10 @@ def create(): peakfind_button.on_click(peakfind_button_callback) def fit_all_button_callback(): - nonlocal det_data - for meas in det_data["Measurements"]: - num_of_peaks = det_data["Measurements"][meas].get("num_of_peaks") + for meas in det_data["Measurements"].values(): + num_of_peaks = meas.get("num_of_peaks") if num_of_peaks is not None and num_of_peaks == 1: - det_data = pyzebra.fitccl( - det_data, + pyzebra.fitccl( meas, guess=[ centre_guess.value, @@ -383,15 +377,13 @@ def create(): fit_all_button.on_click(fit_all_button_callback) def fit_button_callback(): - nonlocal det_data sel_ind = meas_table_source.selected.indices[-1] meas = meas_table_source.data["measurement"][sel_ind] num_of_peaks = det_data["Measurements"][meas].get("num_of_peaks") if num_of_peaks is not None and num_of_peaks == 1: - det_data = pyzebra.fitccl( - det_data, - meas, + pyzebra.fitccl( + det_data["Measurements"][meas], guess=[ centre_guess.value, sigma_guess.value, diff --git a/pyzebra/ccl_findpeaks.py b/pyzebra/ccl_findpeaks.py index adcd614..6153d87 100644 --- a/pyzebra/ccl_findpeaks.py +++ b/pyzebra/ccl_findpeaks.py @@ -5,11 +5,11 @@ from scipy.signal import savgol_filter def ccl_findpeaks( - data, keys, int_threshold=0.8, prominence=50, smooth=False, window_size=7, poly_order=3 + meas, int_threshold=0.8, prominence=50, smooth=False, window_size=7, poly_order=3 ): """function iterates through the dictionary created by load_cclv2 and locates peaks for each measurement - args: data (dictionary from load_cclv2), + args: meas - a single measurement, int_threshold - fraction of threshold_intensity/max_intensity, must be positive num between 0 and 1 i.e. will only detect peaks above 75% of max intensity @@ -29,11 +29,6 @@ def ccl_findpeaks( 'peak_heights': [90.], # height of the peaks (if data vere smoothed its the heigh of the peaks in smoothed data) """ - meas = data["Measurements"][keys] - - if type(data) is not dict and data["file_type"] != "ccl": - print("Data is not a dictionary or was not made from ccl file") - if not 0 <= int_threshold <= 1: int_threshold = 0.8 print( @@ -41,27 +36,27 @@ def ccl_findpeaks( int_threshold, ) - if isinstance(window_size, int) is False or (window_size % 2) == 0 or window_size <= 1: + if not isinstance(window_size, int) or (window_size % 2) == 0 or window_size <= 1: window_size = 7 print( "Invalid value for window_size, select positive odd integer, new value set to!:", window_size, ) - if isinstance(poly_order, int) is False or window_size < poly_order: + if not isinstance(poly_order, int) or window_size < poly_order: poly_order = 3 print( "Invalid value for poly_order, select positive integer smaller than window_size, new value set to:", poly_order, ) - if isinstance(prominence, (int, float)) is False and prominence < 0: + if not isinstance(prominence, (int, float)) and prominence < 0: prominence = 50 print("Invalid value for prominence, select positive number, new value set to:", prominence) omega = meas["om"] counts = np.array(meas["Counts"]) - if smooth is True: + if smooth: itp = interp1d(omega, counts, kind="linear") absintensity = [abs(number) for number in counts] lowest_intensity = min(absintensity) @@ -71,12 +66,10 @@ def ccl_findpeaks( else: smooth_peaks = counts - indexes = sc.signal.find_peaks( + peaks, properties = sc.signal.find_peaks( smooth_peaks, height=int_threshold * max(smooth_peaks), prominence=prominence ) - meas["num_of_peaks"] = len(indexes[0]) - meas["peak_indexes"] = indexes[0] - meas["peak_heights"] = indexes[1]["peak_heights"] + meas["num_of_peaks"] = len(peaks) + meas["peak_indexes"] = peaks + meas["peak_heights"] = properties["peak_heights"] meas["smooth_peaks"] = smooth_peaks # smoothed curve - - return data diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index a106b83..4cb5ce7 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -32,8 +32,7 @@ def create_uncertanities(y, y_err): def fitccl( - data, - keys, + meas, guess, vary, constraints_min, @@ -43,8 +42,7 @@ def fitccl( binning=None, ): """Made for fitting of ccl date where 1 peak is expected. Allows for combination of gaussian and linear model combination - :param data: dictionary after peak fining - :param keys: name of the measurement in the data dict (i.e. M123) + :param meas: measurement in the data dict (i.e. M123) :param guess: initial guess for the fitting, if none, some values are added automatically in order (see below) :param vary: True if parameter can vary during fitting, False if it to be fixed :param numfit_min: minimal value on x axis for numerical integration - if none is centre of gaussian minus 3 sigma @@ -61,8 +59,6 @@ def fitccl( constraints_min = [23, None, 50, 0, 0] constraints_min = [80, None, 1000, 0, 100] """ - meas = data["Measurements"][keys] - if len(meas["peak_indexes"]) > 1: # return in case of more than 1 peaks print("More than 1 peak, measurement skipped") @@ -85,7 +81,7 @@ def fitccl( x = bin_data(x, binning) y = list(meas["Counts"]) y_err = list(np.sqrt(y)) if meas.get("sigma", None) is None else list(meas["sigma"]) - combined = bin_data(create_uncertanities(y,y_err), binning) + combined = bin_data(create_uncertanities(y, y_err), binning) y = [combined[i].n for i in range(len(combined))] y_err = [combined[i].s for i in range(len(combined))] @@ -126,10 +122,10 @@ def fitccl( params = Parameters() params.add_many( ("g_cen", guess[0], bool(vary[0]), np.min(x), np.max(x), None, None), - ("g_width", guess[1], bool(vary[1]), constraints_min[1], constraints_max[1], None, None,), - ("g_amp", guess[2], bool(vary[2]), constraints_min[2], constraints_max[2], None, None,), - ("slope", guess[3], bool(vary[3]), constraints_min[3], constraints_max[3], None, None,), - ("intercept", guess[4], bool(vary[4]), constraints_min[4], constraints_max[4], None, None,), + ("g_width", guess[1], bool(vary[1]), constraints_min[1], constraints_max[1], None, None), + ("g_amp", guess[2], bool(vary[2]), constraints_min[2], constraints_max[2], None, None), + ("slope", guess[3], bool(vary[3]), constraints_min[3], constraints_max[3], None, None), + ("intercept", guess[4], bool(vary[4]), constraints_min[4], constraints_max[4], None, None), ) # the weighted fit result = mod.fit( @@ -225,5 +221,3 @@ def fitccl( d["comps"] = comps d["numfit"] = [numfit_min, numfit_max] meas["fit"] = d - - return data From 3f8900bf87633fb7d2e2ccd7ff81c0b5b6daccc3 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 16 Oct 2020 15:30:52 +0200 Subject: [PATCH 091/116] Rename "Measurements" to "meas" --- pyzebra/app/panel_ccl_integrate.py | 20 ++-- pyzebra/ccl_dict_operation.py | 175 ++++++++++++++--------------- pyzebra/comm_export.py | 2 +- pyzebra/load_1D.py | 2 +- pyzebra/param_study_moduls.py | 34 +++--- 5 files changed, 111 insertions(+), 122 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 7587a24..c8876f8 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -75,7 +75,7 @@ def create(): _, ext = os.path.splitext(new) det_data = pyzebra.parse_1D(file, ext) - meas_list = list(det_data["Measurements"].keys()) + meas_list = list(det_data["meas"].keys()) meas_table_source.data.update(measurement=meas_list, peaks=[0] * len(meas_list)) meas_table_source.selected.indices = [] meas_table_source.selected.indices = [0] @@ -89,7 +89,7 @@ def create(): _, ext = os.path.splitext(upload_button.filename) det_data = pyzebra.parse_1D(file, ext) - meas_list = list(det_data["Measurements"].keys()) + meas_list = list(det_data["meas"].keys()) meas_table_source.data.update(measurement=meas_list, peaks=[0] * len(meas_list)) meas_table_source.selected.indices = [] meas_table_source.selected.indices = [0] @@ -98,14 +98,14 @@ def create(): upload_button.on_change("value", upload_button_callback) def _update_table(): - num_of_peaks = [meas.get("num_of_peaks", 0) for meas in det_data["Measurements"].values()] + num_of_peaks = [meas.get("num_of_peaks", 0) for meas in det_data["meas"].values()] meas_table_source.data.update(peaks=num_of_peaks) def _update_plot(ind): nonlocal peak_pos_textinput_lock peak_pos_textinput_lock = True - meas = det_data["Measurements"][ind] + meas = det_data["meas"][ind] y = meas["Counts"] x = meas["om"] @@ -223,7 +223,7 @@ def create(): if new is not None and not peak_pos_textinput_lock: sel_ind = meas_table_source.selected.indices[-1] meas_name = meas_table_source.data["measurement"][sel_ind] - meas = det_data["Measurements"][meas_name] + meas = det_data["meas"][meas_name] meas["num_of_peaks"] = 1 peak_ind = (np.abs(meas["om"] - float(new))).argmin() @@ -296,7 +296,7 @@ def create(): fit_output_textinput = TextAreaInput(title="Fit results:", width=450, height=400) def peakfind_all_button_callback(): - for meas in det_data["Measurements"].values(): + for meas in det_data["meas"].values(): pyzebra.ccl_findpeaks( meas, int_threshold=peak_int_ratio_spinner.value, @@ -318,7 +318,7 @@ def create(): sel_ind = meas_table_source.selected.indices[-1] meas = meas_table_source.data["measurement"][sel_ind] pyzebra.ccl_findpeaks( - det_data["Measurements"][meas], + det_data["meas"][meas], int_threshold=peak_int_ratio_spinner.value, prominence=peak_prominence_spinner.value, smooth=smooth_toggle.active, @@ -333,7 +333,7 @@ def create(): peakfind_button.on_click(peakfind_button_callback) def fit_all_button_callback(): - for meas in det_data["Measurements"].values(): + for meas in det_data["meas"].values(): num_of_peaks = meas.get("num_of_peaks") if num_of_peaks is not None and num_of_peaks == 1: pyzebra.fitccl( @@ -380,10 +380,10 @@ def create(): sel_ind = meas_table_source.selected.indices[-1] meas = meas_table_source.data["measurement"][sel_ind] - num_of_peaks = det_data["Measurements"][meas].get("num_of_peaks") + num_of_peaks = det_data["meas"][meas].get("num_of_peaks") if num_of_peaks is not None and num_of_peaks == 1: pyzebra.fitccl( - det_data["Measurements"][meas], + det_data["meas"][meas], guess=[ centre_guess.value, sigma_guess.value, diff --git a/pyzebra/ccl_dict_operation.py b/pyzebra/ccl_dict_operation.py index 48fc88e..42b4766 100644 --- a/pyzebra/ccl_dict_operation.py +++ b/pyzebra/ccl_dict_operation.py @@ -12,19 +12,17 @@ def add_dict(dict1, dict2): :return dict1 : combined dictionary Note: dict1 must be made from ccl, otherwise we would have to change the structure of loaded dat file""" - max_measurement_dict1 = max([int(str(keys)[1:]) for keys in dict1["Measurements"]]) + max_measurement_dict1 = max([int(str(keys)[1:]) for keys in dict1["meas"]]) if dict2["meta"]["data_type"] == ".ccl": new_filenames = [ "M" + str(x + max_measurement_dict1) - for x in [int(str(keys)[1:]) for keys in dict2["Measurements"]] + for x in [int(str(keys)[1:]) for keys in dict2["meas"]] ] new_meta_name = "meta" + str(dict2["meta"]["original_filename"]) if new_meta_name not in dict1: - for keys, name in zip(dict2["Measurements"], new_filenames): - dict2["Measurements"][keys]["file_of_origin"] = str( - dict2["meta"]["original_filename"] - ) - dict1["Measurements"][name] = dict2["Measurements"][keys] + for keys, name in zip(dict2["meas"], new_filenames): + dict2["meas"][keys]["file_of_origin"] = str(dict2["meta"]["original_filename"]) + dict1["meas"][name] = dict2["meas"][keys] dict1[new_meta_name] = dict2["meta"] @@ -42,14 +40,14 @@ def add_dict(dict1, dict2): d["h_index"] = float(hkl.split()[-3]) d["k_index"] = float(hkl.split()[-2]) d["l_index"] = float(hkl.split()[-1]) - d["number_of_measurements"] = len(dict2["Measurements"]["NP"]) - d["om"] = dict2["Measurements"]["om"] - d["Counts"] = dict2["Measurements"]["Counts"] - d["monitor"] = dict2["Measurements"]["Monitor1"][0] + d["number_of_measurements"] = len(dict2["meas"]["NP"]) + d["om"] = dict2["meas"]["om"] + d["Counts"] = dict2["meas"]["Counts"] + d["monitor"] = dict2["meas"]["Monitor1"][0] d["temperature"] = dict2["meta"]["temp"] d["mag_field"] = dict2["meta"]["mf"] d["omega_angle"] = dict2["meta"]["omega"] - dict1["Measurements"][new_name] = d + dict1["meas"][new_name] = d print(hkl.split()) for keys in d: print(keys) @@ -82,18 +80,18 @@ def scan_dict(dict): """ d = {} - for i in dict["Measurements"]: - for j in dict["Measurements"]: - if dict["Measurements"][str(i)] != dict["Measurements"][str(j)]: + for i in dict["meas"]: + for j in dict["meas"]: + if dict["meas"][str(i)] != dict["meas"][str(j)]: itup = ( - dict["Measurements"][str(i)]["h_index"], - dict["Measurements"][str(i)]["k_index"], - dict["Measurements"][str(i)]["l_index"], + dict["meas"][str(i)]["h_index"], + dict["meas"][str(i)]["k_index"], + dict["meas"][str(i)]["l_index"], ) jtup = ( - dict["Measurements"][str(j)]["h_index"], - dict["Measurements"][str(j)]["k_index"], - dict["Measurements"][str(j)]["l_index"], + dict["meas"][str(j)]["h_index"], + dict["meas"][str(j)]["k_index"], + dict["meas"][str(j)]["l_index"], ) if itup != jtup: pass @@ -119,47 +117,44 @@ def compare_hkl(dict1, dict2): first dict and M9 in second""" d = {} dupl = 0 - for keys in dict1["Measurements"]: - for key in dict2["Measurements"]: + for keys in dict1["meas"]: + for key in dict2["meas"]: if ( - dict1["Measurements"][str(keys)]["h_index"] - == dict2["Measurements"][str(key)]["h_index"] - and dict1["Measurements"][str(keys)]["k_index"] - == dict2["Measurements"][str(key)]["k_index"] - and dict1["Measurements"][str(keys)]["l_index"] - == dict2["Measurements"][str(key)]["l_index"] + dict1["meas"][str(keys)]["h_index"] == dict2["meas"][str(key)]["h_index"] + and dict1["meas"][str(keys)]["k_index"] == dict2["meas"][str(key)]["k_index"] + and dict1["meas"][str(keys)]["l_index"] == dict2["meas"][str(key)]["l_index"] ): if ( str( ( - str(dict1["Measurements"][str(keys)]["h_index"]) + str(dict1["meas"][str(keys)]["h_index"]) + " " - + str(dict1["Measurements"][str(keys)]["k_index"]) + + str(dict1["meas"][str(keys)]["k_index"]) + " " - + str(dict1["Measurements"][str(keys)]["l_index"]) + + str(dict1["meas"][str(keys)]["l_index"]) ) ) not in d ): d[ str( - str(dict1["Measurements"][str(keys)]["h_index"]) + str(dict1["meas"][str(keys)]["h_index"]) + " " - + str(dict1["Measurements"][str(keys)]["k_index"]) + + str(dict1["meas"][str(keys)]["k_index"]) + " " - + str(dict1["Measurements"][str(keys)]["l_index"]) + + str(dict1["meas"][str(keys)]["l_index"]) ) ] = (str(keys), str(key)) else: dupl = dupl + 1 d[ str( - str(dict1["Measurements"][str(keys)]["h_index"]) + str(dict1["meas"][str(keys)]["h_index"]) + " " - + str(dict1["Measurements"][str(keys)]["k_index"]) + + str(dict1["meas"][str(keys)]["k_index"]) + " " - + str(dict1["Measurements"][str(keys)]["l_index"]) + + str(dict1["meas"][str(keys)]["l_index"]) + "_dupl" + str(dupl) ) @@ -188,13 +183,9 @@ def normalize(dict, key, monitor): :return counts - normalized counts :return sigma - normalized sigma""" - counts = np.array(dict["Measurements"][key]["Counts"]) - sigma = ( - np.sqrt(counts) - if "sigma" not in dict["Measurements"][key] - else dict["Measurements"][key]["sigma"] - ) - monitor_ratio = monitor / dict["Measurements"][key]["monitor"] + counts = np.array(dict["meas"][key]["Counts"]) + sigma = np.sqrt(counts) if "sigma" not in dict["meas"][key] else dict["meas"][key]["sigma"] + monitor_ratio = monitor / dict["meas"][key]["monitor"] scaled_counts = counts * monitor_ratio scaled_sigma = np.array(sigma) * monitor_ratio @@ -212,11 +203,11 @@ def merge(dict1, dict2, keys, auto=True, monitor=100000): note: dict1 and dict2 can be same dict :return dict1 with merged scan""" if auto: - if dict1["Measurements"][keys[0]]["monitor"] == dict2["Measurements"][keys[1]]["monitor"]: - monitor = dict1["Measurements"][keys[0]]["monitor"] + if dict1["meas"][keys[0]]["monitor"] == dict2["meas"][keys[1]]["monitor"]: + monitor = dict1["meas"][keys[0]]["monitor"] # load om and Counts - x1, x2 = dict1["Measurements"][keys[0]]["om"], dict2["Measurements"][keys[1]]["om"] + x1, x2 = dict1["meas"][keys[0]]["om"], dict2["meas"][keys[1]]["om"] cor_y1, y_err1 = normalize(dict1, keys[0], monitor=monitor) cor_y2, y_err2 = normalize(dict2, keys[1], monitor=monitor) # creates touples (om, Counts, sigma) for sorting and further processing @@ -246,21 +237,21 @@ def merge(dict1, dict2, keys, auto=True, monitor=100000): continue if dict1 == dict2: - del dict1["Measurements"][keys[1]] + del dict1["meas"][keys[1]] note = ( f"This measurement was merged with measurement {keys[1]} from " f'file {dict2["meta"]["original_filename"]} \n' ) - if "notes" not in dict1["Measurements"][str(keys[0])]: - dict1["Measurements"][str(keys[0])]["notes"] = note + if "notes" not in dict1["meas"][str(keys[0])]: + dict1["meas"][str(keys[0])]["notes"] = note else: - dict1["Measurements"][str(keys[0])]["notes"] += note + dict1["meas"][str(keys[0])]["notes"] += note - dict1["Measurements"][keys[0]]["om"] = om - dict1["Measurements"][keys[0]]["Counts"] = Counts - dict1["Measurements"][keys[0]]["sigma"] = sigma - dict1["Measurements"][keys[0]]["monitor"] = monitor + dict1["meas"][keys[0]]["om"] = om + dict1["meas"][keys[0]]["Counts"] = Counts + dict1["meas"][keys[0]]["sigma"] = sigma + dict1["meas"][keys[0]]["monitor"] = monitor print("merging done") return dict1 @@ -274,12 +265,12 @@ def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): :arg monitor : final monitor after merging :returns d : dict1 with substracted Counts from dict2 and sigma that comes from the substraction""" - if len(dict1["Measurements"][keys[0]]["om"]) != len(dict2["Measurements"][keys[1]]["om"]): + if len(dict1["meas"][keys[0]]["om"]) != len(dict2["meas"][keys[1]]["om"]): raise ValueError("Omegas have different lengths, cannot be substracted") if auto: - if dict1["Measurements"][keys[0]]["monitor"] == dict2["Measurements"][keys[1]]["monitor"]: - monitor = dict1["Measurements"][keys[0]]["monitor"] + if dict1["meas"][keys[0]]["monitor"] == dict2["meas"][keys[1]]["monitor"]: + monitor = dict1["meas"][keys[0]]["monitor"] cor_y1, y_err1 = normalize(dict1, keys[0], monitor=monitor) cor_y2, y_err2 = normalize(dict2, keys[1], monitor=monitor) @@ -301,17 +292,17 @@ def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): f"{len([num for num in res_nom if num < 0]) / len(res_nom)}" ) - dict1["Measurements"][str(keys[0])]["Counts"] = res_nom - dict1["Measurements"][str(keys[0])]["sigma"] = res_err - dict1["Measurements"][str(keys[0])]["monitor"] = monitor + dict1["meas"][str(keys[0])]["Counts"] = res_nom + dict1["meas"][str(keys[0])]["sigma"] = res_err + dict1["meas"][str(keys[0])]["monitor"] = monitor note = ( f'Measurement {keys[1]} from file {dict2["meta"]["original_filename"]} ' f"was substracted from this measurement \n" ) - if "notes" not in dict1["Measurements"][str(keys[0])]: - dict1["Measurements"][str(keys[0])]["notes"] = note + if "notes" not in dict1["meas"][str(keys[0])]: + dict1["meas"][str(keys[0])]["notes"] = note else: - dict1["Measurements"][str(keys[0])]["notes"] += note + dict1["meas"][str(keys[0])]["notes"] += note return dict1 @@ -380,24 +371,24 @@ def compare_dict(dict1, dict2): # compare Measurements S.append( "Number of measurements in %s = %s \n" - % (dict1["meta"]["original_filename"], len(dict1["Measurements"])) + % (dict1["meta"]["original_filename"], len(dict1["meas"])) ) S.append( "Number of measurements in %s = %s \n" - % (dict2["meta"]["original_filename"], len(dict2["Measurements"])) + % (dict2["meta"]["original_filename"], len(dict2["meas"])) ) S.append("Different values in Measurements:\n") select_set = ["om", "Counts", "sigma"] exlude_set = ["time", "Counts", "date", "notes"] for keys1 in comp: - for key2 in dict1["Measurements"][str(comp[str(keys1)][0])]: + for key2 in dict1["meas"][str(comp[str(keys1)][0])]: if key2 in exlude_set: continue if key2 not in select_set: try: if ( - dict1["Measurements"][comp[str(keys1)][0]][str(key2)] - != dict2["Measurements"][str(comp[str(keys1)][1])][str(key2)] + dict1["meas"][comp[str(keys1)][0]][str(key2)] + != dict2["meas"][str(comp[str(keys1)][1])][str(key2)] ): S.append( "Measurement value " @@ -407,21 +398,21 @@ def compare_dict(dict1, dict2): ) S.append( " dict1: %s \n" - % str(dict1["Measurements"][comp[str(keys1)][0]][str(key2)]) + % str(dict1["meas"][comp[str(keys1)][0]][str(key2)]) ) S.append( " dict2: %s \n" - % str(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) + % str(dict2["meas"][comp[str(keys1)][1]][str(key2)]) ) if key2 not in conflicts: conflicts[key2] = {} conflicts[key2]["amount"] = 1 - conflicts[key2]["measurements"] = str(comp[str(keys1)]) + conflicts[key2]["meas"] = str(comp[str(keys1)]) else: conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 - conflicts[key2]["measurements"] = ( - conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)])) + conflicts[key2]["meas"] = ( + conflicts[key2]["meas"] + " " + (str(comp[str(keys1)])) ) except KeyError as e: print("Missing keys, some files were probably merged or substracted") @@ -429,11 +420,11 @@ def compare_dict(dict1, dict2): else: try: - comparison = list( - dict1["Measurements"][comp[str(keys1)][0]][str(key2)] - ) == list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) - if len(list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)])) != len( - list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)]) + comparison = list(dict1["meas"][comp[str(keys1)][0]][str(key2)]) == list( + dict2["meas"][comp[str(keys1)][1]][str(key2)] + ) + if len(list(dict1["meas"][comp[str(keys1)][0]][str(key2)])) != len( + list(dict2["meas"][comp[str(keys1)][1]][str(key2)]) ): if str("different length of %s" % key2) not in warnings: warnings[str("different length of %s" % key2)] = list() @@ -453,20 +444,20 @@ def compare_dict(dict1, dict2): ) S.append( " dict1: %s \n" - % str(list(dict1["Measurements"][comp[str(keys1)][0]][str(key2)])) + % str(list(dict1["meas"][comp[str(keys1)][0]][str(key2)])) ) S.append( " dict2: %s \n" - % str(list(dict2["Measurements"][comp[str(keys1)][1]][str(key2)])) + % str(list(dict2["meas"][comp[str(keys1)][1]][str(key2)])) ) if key2 not in conflicts: conflicts[key2] = {} conflicts[key2]["amount"] = 1 - conflicts[key2]["measurements"] = str(comp[str(keys1)]) + conflicts[key2]["meas"] = str(comp[str(keys1)]) else: conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 - conflicts[key2]["measurements"] = ( - conflicts[key2]["measurements"] + " " + (str(comp[str(keys1)])) + conflicts[key2]["meas"] = ( + conflicts[key2]["meas"] + " " + (str(comp[str(keys1)])) ) except KeyError as e: print("Missing keys, some files were probably merged or substracted") @@ -474,7 +465,7 @@ def compare_dict(dict1, dict2): for keys in conflicts: try: - conflicts[str(keys)]["measurements"] = conflicts[str(keys)]["measurements"].split(" ") + conflicts[str(keys)]["meas"] = conflicts[str(keys)]["meas"].split(" ") except: continue report_string = "".join(S) @@ -489,18 +480,18 @@ def guess_next(dict1, dict2, comp): if ( abs( ( - dict1["Measurements"][str(comp[keys][0])]["temperature"] - - dict2["Measurements"][str(comp[keys][1])]["temperature"] + dict1["meas"][str(comp[keys][0])]["temperature"] + - dict2["meas"][str(comp[keys][1])]["temperature"] ) - / dict2["Measurements"][str(comp[keys][1])]["temperature"] + / dict2["meas"][str(comp[keys][1])]["temperature"] ) < threshold and abs( ( - dict1["Measurements"][str(comp[keys][0])]["mag_field"] - - dict2["Measurements"][str(comp[keys][1])]["mag_field"] + dict1["meas"][str(comp[keys][0])]["mag_field"] + - dict2["meas"][str(comp[keys][1])]["mag_field"] ) - / dict2["Measurements"][str(comp[keys][1])]["mag_field"] + / dict2["meas"][str(comp[keys][1])]["mag_field"] ) < threshold ): diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py index cb70126..59af5a7 100644 --- a/pyzebra/comm_export.py +++ b/pyzebra/comm_export.py @@ -30,7 +30,7 @@ def export_comm(data, path, lorentz=False): padding = [4, 6, 10, 8] with open(str(path + extension), "w") as out_file: - for keys, meas in data["Measurements"].items(): + for keys, meas in data["meas"].items(): if "fit" not in meas: print("Measurement skipped - no fit value for:", keys) continue diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 7bac265..c5e9856 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -183,4 +183,4 @@ def parse_1D(fileobj, data_type): metadata["data_type"] = data_type metadata["area_method"] = "fit" - return {"meta": metadata, "Measurements": measurements} + return {"meta": metadata, "meas": measurements} diff --git a/pyzebra/param_study_moduls.py b/pyzebra/param_study_moduls.py index e04afe1..ea9280a 100644 --- a/pyzebra/param_study_moduls.py +++ b/pyzebra/param_study_moduls.py @@ -38,12 +38,12 @@ def load_dats(filepath): dict1 = add_dict(dict1, load_1D(file_list[i][0])) else: dict1 = add_dict(dict1, load_1D(file_list[i])) - dict1["Measurements"][str("M" + str(i + 1))]["params"] = {} + dict1["meas"][str("M" + str(i + 1))]["params"] = {} if data_type == "txt": for x in range(len(col_names) - 1): - dict1["Measurements"][str("M" + str(i + 1))]["params"][ - col_names[x + 1] - ] = file_list[i][x + 1] + dict1["meas"][str("M" + str(i + 1))]["params"][col_names[x + 1]] = file_list[i][ + x + 1 + ] return dict1 @@ -55,7 +55,7 @@ def create_dataframe(dict1): # create dictionary to which we pull only wanted items before transforming it to pd.dataframe pull_dict = {} pull_dict["filenames"] = list() - for key in dict1["Measurements"]["M1"]["params"]: + for key in dict1["meas"]["M1"]["params"]: pull_dict[key] = list() pull_dict["temperature"] = list() pull_dict["mag_field"] = list() @@ -65,21 +65,19 @@ def create_dataframe(dict1): pull_dict["Counts"] = list() # populate the dict - for keys in dict1["Measurements"]: - if "file_of_origin" in dict1["Measurements"][keys]: - pull_dict["filenames"].append( - dict1["Measurements"][keys]["file_of_origin"].split("/")[-1] - ) + for keys in dict1["meas"]: + if "file_of_origin" in dict1["meas"][keys]: + pull_dict["filenames"].append(dict1["meas"][keys]["file_of_origin"].split("/")[-1]) else: pull_dict["filenames"].append(dict1["meta"]["original_filename"].split("/")[-1]) - for key in dict1["Measurements"][keys]["params"]: - pull_dict[str(key)].append(float(dict1["Measurements"][keys]["params"][key])) - pull_dict["temperature"].append(dict1["Measurements"][keys]["temperature"]) - pull_dict["mag_field"].append(dict1["Measurements"][keys]["mag_field"]) - pull_dict["fit_area"].append(dict1["Measurements"][keys]["fit"]["fit_area"]) - pull_dict["int_area"].append(dict1["Measurements"][keys]["fit"]["int_area"]) - pull_dict["om"].append(dict1["Measurements"][keys]["om"]) - pull_dict["Counts"].append(dict1["Measurements"][keys]["Counts"]) + for key in dict1["meas"][keys]["params"]: + pull_dict[str(key)].append(float(dict1["meas"][keys]["params"][key])) + pull_dict["temperature"].append(dict1["meas"][keys]["temperature"]) + pull_dict["mag_field"].append(dict1["meas"][keys]["mag_field"]) + pull_dict["fit_area"].append(dict1["meas"][keys]["fit"]["fit_area"]) + pull_dict["int_area"].append(dict1["meas"][keys]["fit"]["int_area"]) + pull_dict["om"].append(dict1["meas"][keys]["om"]) + pull_dict["Counts"].append(dict1["meas"][keys]["Counts"]) return pd.DataFrame(data=pull_dict) From 23c8a573d766eb2f48c58da8e5152b668529876a Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 16 Oct 2020 15:53:30 +0200 Subject: [PATCH 092/116] Use integer keys for measurements --- pyzebra/comm_export.py | 6 +++--- pyzebra/load_1D.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py index 59af5a7..3ab2324 100644 --- a/pyzebra/comm_export.py +++ b/pyzebra/comm_export.py @@ -30,11 +30,11 @@ def export_comm(data, path, lorentz=False): padding = [4, 6, 10, 8] with open(str(path + extension), "w") as out_file: - for keys, meas in data["meas"].items(): + for key, meas in data["meas"].items(): if "fit" not in meas: - print("Measurement skipped - no fit value for:", keys) + print("Measurement skipped - no fit value for:", key) continue - meas_number_str = f"{keys[1:]:{align}{padding[0]}}" + meas_number_str = f"{key:{align}{padding[0]}}" h_str = f'{int(meas["h_index"]):{padding[1]}}' k_str = f'{int(meas["k_index"]):{padding[1]}}' l_str = f'{int(meas["l_index"]):{padding[1]}}' diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index c5e9856..cb2ef7a 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -152,7 +152,7 @@ def parse_1D(fileobj, data_type): int(next_line.split()[0]), ) d["Counts"] = counts - measurements[str("M" + str(measurement_number))] = d + measurements[measurement_number] = d if all(decimal): metadata["indices"] = "hkl" From 6337afb4c363d18c0dfa4bb17504c02310e7bdaa Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 16 Oct 2020 16:43:43 +0200 Subject: [PATCH 093/116] Allow fitting of measurements without peaks --- pyzebra/app/panel_ccl_integrate.py | 107 +++++++++++++++-------------- 1 file changed, 56 insertions(+), 51 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index c8876f8..7fd580f 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -154,8 +154,17 @@ def create(): (params["g_amp"].value - fit["int_area"].n) / params["g_amp"].value, ) ) - numfit_min_span.location = x[fit["numfit"][0]] - numfit_max_span.location = x[fit["numfit"][1]] + numfit_min, numfit_max = fit["numfit"] + if numfit_min is None: + numfit_min_span.location = None + else: + numfit_min_span.location = x[numfit_min] + + if numfit_max is None: + numfit_max_span.location = None + else: + numfit_max_span.location = x[numfit_max] + else: plot_gauss_source.data.update(x=[], y=[]) plot_bkg_source.data.update(x=[], y=[]) @@ -334,56 +343,8 @@ def create(): def fit_all_button_callback(): for meas in det_data["meas"].values(): - num_of_peaks = meas.get("num_of_peaks") - if num_of_peaks is not None and num_of_peaks == 1: - pyzebra.fitccl( - meas, - guess=[ - centre_guess.value, - sigma_guess.value, - ampl_guess.value, - slope_guess.value, - offset_guess.value, - ], - vary=[ - centre_vary.active, - sigma_vary.active, - ampl_vary.active, - slope_vary.active, - offset_vary.active, - ], - constraints_min=[ - centre_min.value, - sigma_min.value, - ampl_min.value, - slope_min.value, - offset_min.value, - ], - constraints_max=[ - centre_max.value, - sigma_max.value, - ampl_max.value, - slope_max.value, - offset_max.value, - ], - numfit_min=integ_from.value, - numfit_max=integ_to.value, - ) - - sel_ind = meas_table_source.selected.indices[-1] - _update_plot(meas_table_source.data["measurement"][sel_ind]) - - fit_all_button = Button(label="Fit All", button_type="primary", default_size=145) - fit_all_button.on_click(fit_all_button_callback) - - def fit_button_callback(): - sel_ind = meas_table_source.selected.indices[-1] - meas = meas_table_source.data["measurement"][sel_ind] - - num_of_peaks = det_data["meas"][meas].get("num_of_peaks") - if num_of_peaks is not None and num_of_peaks == 1: pyzebra.fitccl( - det_data["meas"][meas], + meas, guess=[ centre_guess.value, sigma_guess.value, @@ -412,8 +373,52 @@ def create(): slope_max.value, offset_max.value, ], + numfit_min=integ_from.value, + numfit_max=integ_to.value, ) + sel_ind = meas_table_source.selected.indices[-1] + _update_plot(meas_table_source.data["measurement"][sel_ind]) + + fit_all_button = Button(label="Fit All", button_type="primary", default_size=145) + fit_all_button.on_click(fit_all_button_callback) + + def fit_button_callback(): + sel_ind = meas_table_source.selected.indices[-1] + meas = meas_table_source.data["measurement"][sel_ind] + + pyzebra.fitccl( + det_data["meas"][meas], + guess=[ + centre_guess.value, + sigma_guess.value, + ampl_guess.value, + slope_guess.value, + offset_guess.value, + ], + vary=[ + centre_vary.active, + sigma_vary.active, + ampl_vary.active, + slope_vary.active, + offset_vary.active, + ], + constraints_min=[ + centre_min.value, + sigma_min.value, + ampl_min.value, + slope_min.value, + offset_min.value, + ], + constraints_max=[ + centre_max.value, + sigma_max.value, + ampl_max.value, + slope_max.value, + offset_max.value, + ], + ) + _update_plot(meas) fit_button = Button(label="Fit Current", default_size=145) From baf4a437d309989e1c924dd6c0cedccfc832f631 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Fri, 16 Oct 2020 16:44:25 +0200 Subject: [PATCH 094/116] Fix integ values not applying in Fit Current --- pyzebra/app/panel_ccl_integrate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 7fd580f..df73f88 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -417,6 +417,8 @@ def create(): slope_max.value, offset_max.value, ], + numfit_min=integ_from.value, + numfit_max=integ_to.value, ) _update_plot(meas) From 8104c54302953b9279a6b4912fc97c53329c8868 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Mon, 19 Oct 2020 14:19:33 +0200 Subject: [PATCH 095/116] Fix issue reading ccl files with 5 digit counts --- pyzebra/load_1D.py | 105 +++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 57 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index cb2ef7a..27d2786 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -99,65 +99,56 @@ def parse_1D(fileobj, data_type): if data_type == ".ccl": measurements = {} decimal = list() - data = fileobj.readlines() - position = -1 - for lines in data: - position = position + 1 - if ( - bool(re.match("(\s\s\s\d)", lines[0:4])) == True - or bool(re.match("(\s\s\d\d)", lines[0:4])) == True - or bool(re.match("(\s\d\d\d)", lines[0:4])) == True - or bool(re.match("(\d\d\d\d)", lines[0:4])) == True + for line in fileobj: + counts = [] + measurement_number = int(line.split()[0]) + d = {} + d["h_index"] = float(line.split()[1]) + decimal.append(bool(Decimal(d["h_index"]) % 1 == 0)) + d["k_index"] = float(line.split()[2]) + decimal.append(bool(Decimal(d["k_index"]) % 1 == 0)) + d["l_index"] = float(line.split()[3]) + decimal.append(bool(Decimal(d["l_index"]) % 1 == 0)) + if metadata["zebra_mode"] == "bi": + d["twotheta_angle"] = float(line.split()[4]) # gamma + d["omega_angle"] = float(line.split()[5]) # omega + d["chi_angle"] = float(line.split()[6]) # nu + d["phi_angle"] = float(line.split()[7]) # doesnt matter + elif metadata["zebra_mode"] == "nb": + d["gamma_angle"] = float(line.split()[4]) # gamma + d["omega_angle"] = float(line.split()[5]) # omega + d["nu_angle"] = float(line.split()[6]) # nu + d["unkwn_angle"] = float(line.split()[7]) + + next_line = next(fileobj) + d["number_of_measurements"] = int(next_line.split()[0]) + d["angle_step"] = float(next_line.split()[1]) + d["monitor"] = float(next_line.split()[2]) + d["temperature"] = float(next_line.split()[3]) + d["mag_field"] = float(next_line.split()[4]) + d["date"] = str(next_line.split()[5]) + d["time"] = str(next_line.split()[6]) + d["scan_type"] = str(next_line.split()[7]) + for i in range( + int(int(next_line.split()[0]) / 10) + (int(next_line.split()[0]) % 10 > 0) ): - counts = [] - measurement_number = int(lines.split()[0]) - d = {} - d["h_index"] = float(lines.split()[1]) - decimal.append(bool(Decimal(d["h_index"]) % 1 == 0)) - d["k_index"] = float(lines.split()[2]) - decimal.append(bool(Decimal(d["k_index"]) % 1 == 0)) - d["l_index"] = float(lines.split()[3]) - decimal.append(bool(Decimal(d["l_index"]) % 1 == 0)) - if metadata["zebra_mode"] == "bi": - d["twotheta_angle"] = float(lines.split()[4]) # gamma - d["omega_angle"] = float(lines.split()[5]) # omega - d["chi_angle"] = float(lines.split()[6]) # nu - d["phi_angle"] = float(lines.split()[7]) # doesnt matter - elif metadata["zebra_mode"] == "nb": - d["gamma_angle"] = float(lines.split()[4]) # gamma - d["omega_angle"] = float(lines.split()[5]) # omega - d["nu_angle"] = float(lines.split()[6]) # nu - d["unkwn_angle"] = float(lines.split()[7]) + fileline = next(fileobj).split() + numbers = [int(w) for w in fileline] + counts = counts + numbers + d["om"] = np.linspace( + float(line.split()[5]) + - (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), + float(line.split()[5]) + + (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), + int(next_line.split()[0]), + ) + d["Counts"] = counts + measurements[measurement_number] = d - next_line = data[position + 1] - d["number_of_measurements"] = int(next_line.split()[0]) - d["angle_step"] = float(next_line.split()[1]) - d["monitor"] = float(next_line.split()[2]) - d["temperature"] = float(next_line.split()[3]) - d["mag_field"] = float(next_line.split()[4]) - d["date"] = str(next_line.split()[5]) - d["time"] = str(next_line.split()[6]) - d["scan_type"] = str(next_line.split()[7]) - for i in range( - int(int(next_line.split()[0]) / 10) + (int(next_line.split()[0]) % 10 > 0) - ): - fileline = data[position + 2 + i].split() - numbers = [int(w) for w in fileline] - counts = counts + numbers - d["om"] = np.linspace( - float(lines.split()[5]) - - (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), - float(lines.split()[5]) - + (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), - int(next_line.split()[0]), - ) - d["Counts"] = counts - measurements[measurement_number] = d - - if all(decimal): - metadata["indices"] = "hkl" - else: - metadata["indices"] = "real" + if all(decimal): + metadata["indices"] = "hkl" + else: + metadata["indices"] = "real" elif data_type == ".dat": # skip the first 2 rows, the third row contans the column names From f62aab74b7173a7d57342b31ca88b48c1b76373d Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Mon, 19 Oct 2020 15:05:31 +0200 Subject: [PATCH 096/116] Update load_1D.py changed the dat part --- pyzebra/load_1D.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 27d2786..02b7466 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -96,8 +96,8 @@ def parse_1D(fileobj, data_type): break # read data + measurements = {} if data_type == ".ccl": - measurements = {} decimal = list() for line in fileobj: counts = [] @@ -155,8 +155,8 @@ def parse_1D(fileobj, data_type): next(fileobj) next(fileobj) col_names = next(fileobj).split() - data_cols = defaultdict(list) + for line in fileobj: if "END-OF-DATA" in line: # this is the end of data @@ -165,7 +165,15 @@ def parse_1D(fileobj, data_type): for name, val in zip(col_names, line.split()): data_cols[name].append(float(val)) - measurements = dict(data_cols) + data_cols['h_index'] = float(metadata['title'].split()[-3]) + data_cols['k_index'] = float(metadata['title'].split()[-2]) + data_cols['l_index'] = float(metadata['title'].split()[-1]) + data_cols['temperature'] = metadata['temp'] + data_cols['mag_field'] = metadata['mf'] + data_cols['omega_angle'] = metadata['omega'] + data_cols['number_of_measurements'] = len(data_cols['om']) + data_cols['monitor'] = data_cols['Monitor1'][0] + measurements["1"] = dict(data_cols) else: print("Unknown file extention") From b8731b9e01e996c54bbdba1f52771701ea3e00a2 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Mon, 19 Oct 2020 15:11:03 +0200 Subject: [PATCH 097/116] Refactor parsing of the first lines in each meas --- pyzebra/load_1D.py | 104 ++++++++++++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 35 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 02b7466..952afff 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -57,6 +57,40 @@ META_VARS_FLOAT = ( ) META_UB_MATRIX = ("ub1j", "ub2j", "ub3j") +CCL_FIRST_LINE = ( + # the first element is `measurement_number`, which we don't save to metadata + ("h_index", float), + ("k_index", float), + ("l_index", float), +) + +CCL_FIRST_LINE_BI = ( + *CCL_FIRST_LINE, + ("twotheta_angle", float), + ("omega_angle", float), + ("chi_angle", float), + ("phi_angle", float), +) + +CCL_FIRST_LINE_NB = ( + *CCL_FIRST_LINE, + ("gamma_angle", float), + ("omega_angle", float), + ("nu_angle", float), + ("unkwn_angle", float), +) + +CCL_SECOND_LINE = ( + ("number_of_measurements", int), + ("angle_step", float), + ("monitor", float), + ("temperature", float), + ("mag_field", float), + ("date", str), + ("time", str), + ("scan_type", str), +) + def load_1D(filepath): """ @@ -99,42 +133,31 @@ def parse_1D(fileobj, data_type): measurements = {} if data_type == ".ccl": decimal = list() - for line in fileobj: - counts = [] - measurement_number = int(line.split()[0]) - d = {} - d["h_index"] = float(line.split()[1]) - decimal.append(bool(Decimal(d["h_index"]) % 1 == 0)) - d["k_index"] = float(line.split()[2]) - decimal.append(bool(Decimal(d["k_index"]) % 1 == 0)) - d["l_index"] = float(line.split()[3]) - decimal.append(bool(Decimal(d["l_index"]) % 1 == 0)) - if metadata["zebra_mode"] == "bi": - d["twotheta_angle"] = float(line.split()[4]) # gamma - d["omega_angle"] = float(line.split()[5]) # omega - d["chi_angle"] = float(line.split()[6]) # nu - d["phi_angle"] = float(line.split()[7]) # doesnt matter - elif metadata["zebra_mode"] == "nb": - d["gamma_angle"] = float(line.split()[4]) # gamma - d["omega_angle"] = float(line.split()[5]) # omega - d["nu_angle"] = float(line.split()[6]) # nu - d["unkwn_angle"] = float(line.split()[7]) + if metadata["zebra_mode"] == "bi": + ccl_first_line = CCL_FIRST_LINE_BI + elif metadata["zebra_mode"] == "nb": + ccl_first_line = CCL_FIRST_LINE_NB + ccl_second_line = CCL_SECOND_LINE + + for line in fileobj: + d = {} + + # first line + measurement_number, *params = line.split() + for param, (param_name, param_type) in zip(params, ccl_first_line): + d[param_name] = param_type(param) + + decimal.append(bool(Decimal(d["h_index"]) % 1 == 0)) + decimal.append(bool(Decimal(d["k_index"]) % 1 == 0)) + decimal.append(bool(Decimal(d["l_index"]) % 1 == 0)) + + # second line next_line = next(fileobj) - d["number_of_measurements"] = int(next_line.split()[0]) - d["angle_step"] = float(next_line.split()[1]) - d["monitor"] = float(next_line.split()[2]) - d["temperature"] = float(next_line.split()[3]) - d["mag_field"] = float(next_line.split()[4]) - d["date"] = str(next_line.split()[5]) - d["time"] = str(next_line.split()[6]) - d["scan_type"] = str(next_line.split()[7]) - for i in range( - int(int(next_line.split()[0]) / 10) + (int(next_line.split()[0]) % 10 > 0) - ): - fileline = next(fileobj).split() - numbers = [int(w) for w in fileline] - counts = counts + numbers + params = next_line.split() + for param, (param_name, param_type) in zip(params, ccl_second_line): + d[param_name] = param_type(param) + d["om"] = np.linspace( float(line.split()[5]) - (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), @@ -142,8 +165,19 @@ def parse_1D(fileobj, data_type): + (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), int(next_line.split()[0]), ) + + # subsequent lines with counts + counts = [] + for i in range( + int(int(next_line.split()[0]) / 10) + (int(next_line.split()[0]) % 10 > 0) + ): + fileline = next(fileobj).split() + numbers = [int(w) for w in fileline] + counts = counts + numbers + d["Counts"] = counts - measurements[measurement_number] = d + + measurements[int(measurement_number)] = d if all(decimal): metadata["indices"] = "hkl" From 18ba623fd84d8f9707886e8d7b6a5868e14337b5 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Mon, 19 Oct 2020 15:41:55 +0200 Subject: [PATCH 098/116] Refactor read of counts and clarify d["om"] --- pyzebra/load_1D.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 952afff..44dbcd5 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -159,22 +159,15 @@ def parse_1D(fileobj, data_type): d[param_name] = param_type(param) d["om"] = np.linspace( - float(line.split()[5]) - - (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), - float(line.split()[5]) - + (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), - int(next_line.split()[0]), + d["omega_angle"] - (d["number_of_measurements"] / 2) * d["angle_step"], + d["omega_angle"] + (d["number_of_measurements"] / 2) * d["angle_step"], + d["number_of_measurements"], ) # subsequent lines with counts counts = [] - for i in range( - int(int(next_line.split()[0]) / 10) + (int(next_line.split()[0]) % 10 > 0) - ): - fileline = next(fileobj).split() - numbers = [int(w) for w in fileline] - counts = counts + numbers - + while len(counts) < d["number_of_measurements"]: + counts.extend(map(int, next(fileobj).split())) d["Counts"] = counts measurements[int(measurement_number)] = d From 0156752686a87a01e878280b7fd3507f738b4ee7 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 20 Oct 2020 10:16:37 +0200 Subject: [PATCH 099/116] Add debug script --- .vscode/launch.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..46b92fb --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,13 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "pyzebra", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/pyzebra/cli.py", + "console": "internalConsole", + "env": {}, + }, + ] +} From 6722ee601e925575c8f1e5308a235305ed71f5f1 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Tue, 20 Oct 2020 10:44:01 +0200 Subject: [PATCH 100/116] Update fit2.py moved the check up so it catches it right after fitting. I think this should solve the problem. --- pyzebra/fit2.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 4cb5ce7..54a463c 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -131,14 +131,16 @@ def fitccl( result = mod.fit( y, params, weights=[np.abs(1 / y_err[i]) for i in range(len(y_err))], x=x, calc_covar=True ) - # u.ufloat to work with uncertanities - fit_area = u.ufloat(result.params["g_amp"].value, result.params["g_amp"].stderr) - comps = result.eval_components() - + if result.params["g_amp"].stderr is None: result.params["g_amp"].stderr = result.params["g_amp"].value elif result.params["g_amp"].stderr > result.params["g_amp"].value: result.params["g_amp"].stderr = result.params["g_amp"].value + # u.ufloat to work with uncertanities + fit_area = u.ufloat(result.params["g_amp"].value, result.params["g_amp"].stderr) + comps = result.eval_components() + + if len(meas["peak_indexes"]) == 0: # for case of no peak, there is no reason to integrate, therefore fit and int are equal From 48efa93234cd76553a11a2441e27c676ae915826 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 13 Oct 2020 14:09:27 +0200 Subject: [PATCH 101/116] Display omega in hovertool --- pyzebra/app/panel_hdf_viewer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyzebra/app/panel_hdf_viewer.py b/pyzebra/app/panel_hdf_viewer.py index c0e1409..956506a 100644 --- a/pyzebra/app/panel_hdf_viewer.py +++ b/pyzebra/app/panel_hdf_viewer.py @@ -87,7 +87,8 @@ def create(): temperature_spinner.value = det_data["temperature"][index] gamma, nu = calculate_pol(det_data, index) - image_source.data.update(gamma=[gamma], nu=[nu]) + omega = det_data["rot_angle"][index] + image_source.data.update(gamma=[gamma], nu=[nu], omega=[omega]) def update_overview_plot(): h5_data = det_data["data"] @@ -161,6 +162,7 @@ def create(): l=[np.zeros((1, 1))], gamma=[np.zeros((1, 1))], nu=[np.zeros((1, 1))], + omega=[np.zeros((1, 1))], x=[0], y=[0], dw=[IMAGE_W], @@ -173,12 +175,14 @@ def create(): l_glyph = Image(image="l", x="x", y="y", dw="dw", dh="dh", global_alpha=0) gamma_glyph = Image(image="gamma", x="x", y="y", dw="dw", dh="dh", global_alpha=0) nu_glyph = Image(image="nu", x="x", y="y", dw="dw", dh="dh", global_alpha=0) + omega_glyph = Image(image="omega", x="x", y="y", dw="dw", dh="dh", global_alpha=0) plot.add_glyph(image_source, h_glyph) plot.add_glyph(image_source, k_glyph) plot.add_glyph(image_source, l_glyph) plot.add_glyph(image_source, gamma_glyph) plot.add_glyph(image_source, nu_glyph) + plot.add_glyph(image_source, omega_glyph) image_glyph = Image(image="image", x="x", y="y", dw="dw", dh="dh") plot.add_glyph(image_source, image_glyph, name="image_glyph") @@ -224,6 +228,7 @@ def create(): ("intensity", "@image"), ("gamma", "@gamma"), ("nu", "@nu"), + ("omega", "@omega"), ("h", "@h"), ("k", "@k"), ("l", "@l"), From b07707b37d251e5d074ceedd225cbde78295ce69 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 20 Oct 2020 11:57:22 +0200 Subject: [PATCH 102/116] Don't stop on measurements with failed fit Fix #13 --- pyzebra/fit2.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 54a463c..242149a 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -128,20 +128,22 @@ def fitccl( ("intercept", guess[4], bool(vary[4]), constraints_min[4], constraints_max[4], None, None), ) # the weighted fit - result = mod.fit( - y, params, weights=[np.abs(1 / y_err[i]) for i in range(len(y_err))], x=x, calc_covar=True - ) - + try: + result = mod.fit( + y, params, weights=[np.abs(1 / val) for val in y_err], x=x, calc_covar=True, + ) + except ValueError: + return + if result.params["g_amp"].stderr is None: result.params["g_amp"].stderr = result.params["g_amp"].value elif result.params["g_amp"].stderr > result.params["g_amp"].value: result.params["g_amp"].stderr = result.params["g_amp"].value + # u.ufloat to work with uncertanities fit_area = u.ufloat(result.params["g_amp"].value, result.params["g_amp"].stderr) comps = result.eval_components() - - if len(meas["peak_indexes"]) == 0: # for case of no peak, there is no reason to integrate, therefore fit and int are equal int_area = fit_area From 3980e7abd086036d3fe3f23faed5f821ed67e097 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 20 Oct 2020 13:54:24 +0200 Subject: [PATCH 103/116] Add fit results to the table --- pyzebra/app/panel_ccl_integrate.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index df73f88..c9e840d 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -76,7 +76,9 @@ def create(): det_data = pyzebra.parse_1D(file, ext) meas_list = list(det_data["meas"].keys()) - meas_table_source.data.update(measurement=meas_list, peaks=[0] * len(meas_list)) + meas_table_source.data.update( + measurement=meas_list, peaks=[0] * len(meas_list), fit=[0] * len(meas_list) + ) meas_table_source.selected.indices = [] meas_table_source.selected.indices = [0] @@ -90,7 +92,9 @@ def create(): det_data = pyzebra.parse_1D(file, ext) meas_list = list(det_data["meas"].keys()) - meas_table_source.data.update(measurement=meas_list, peaks=[0] * len(meas_list)) + meas_table_source.data.update( + measurement=meas_list, peaks=[0] * len(meas_list), fit=[0] * len(meas_list) + ) meas_table_source.selected.indices = [] meas_table_source.selected.indices = [0] @@ -99,7 +103,8 @@ def create(): def _update_table(): num_of_peaks = [meas.get("num_of_peaks", 0) for meas in det_data["meas"].values()] - meas_table_source.data.update(peaks=num_of_peaks) + fit_ok = [(1 if "fit" in meas else 0) for meas in det_data["meas"].values()] + meas_table_source.data.update(peaks=num_of_peaks, fit=fit_ok) def _update_plot(ind): nonlocal peak_pos_textinput_lock @@ -215,14 +220,15 @@ def create(): if new: _update_plot(meas_table_source.data["measurement"][new[-1]]) - meas_table_source = ColumnDataSource(dict(measurement=[], peaks=[])) + meas_table_source = ColumnDataSource(dict(measurement=[], peaks=[], fit=[])) meas_table = DataTable( source=meas_table_source, columns=[ TableColumn(field="measurement", title="Meas"), TableColumn(field="peaks", title="Peaks"), + TableColumn(field="fit", title="Fit"), ], - width=100, + width=150, index_position=None, ) @@ -379,6 +385,7 @@ def create(): sel_ind = meas_table_source.selected.indices[-1] _update_plot(meas_table_source.data["measurement"][sel_ind]) + _update_table() fit_all_button = Button(label="Fit All", button_type="primary", default_size=145) fit_all_button.on_click(fit_all_button_callback) @@ -422,6 +429,7 @@ def create(): ) _update_plot(meas) + _update_table() fit_button = Button(label="Fit Current", default_size=145) fit_button.on_click(fit_button_callback) From 8361fac5c30ab705d7015bd51e43bf2064bbb99a Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Wed, 21 Oct 2020 13:25:26 +0200 Subject: [PATCH 104/116] Revert "Display omega in hovertool" This reverts commit 48efa93234cd76553a11a2441e27c676ae915826. --- pyzebra/app/panel_hdf_viewer.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pyzebra/app/panel_hdf_viewer.py b/pyzebra/app/panel_hdf_viewer.py index 956506a..c0e1409 100644 --- a/pyzebra/app/panel_hdf_viewer.py +++ b/pyzebra/app/panel_hdf_viewer.py @@ -87,8 +87,7 @@ def create(): temperature_spinner.value = det_data["temperature"][index] gamma, nu = calculate_pol(det_data, index) - omega = det_data["rot_angle"][index] - image_source.data.update(gamma=[gamma], nu=[nu], omega=[omega]) + image_source.data.update(gamma=[gamma], nu=[nu]) def update_overview_plot(): h5_data = det_data["data"] @@ -162,7 +161,6 @@ def create(): l=[np.zeros((1, 1))], gamma=[np.zeros((1, 1))], nu=[np.zeros((1, 1))], - omega=[np.zeros((1, 1))], x=[0], y=[0], dw=[IMAGE_W], @@ -175,14 +173,12 @@ def create(): l_glyph = Image(image="l", x="x", y="y", dw="dw", dh="dh", global_alpha=0) gamma_glyph = Image(image="gamma", x="x", y="y", dw="dw", dh="dh", global_alpha=0) nu_glyph = Image(image="nu", x="x", y="y", dw="dw", dh="dh", global_alpha=0) - omega_glyph = Image(image="omega", x="x", y="y", dw="dw", dh="dh", global_alpha=0) plot.add_glyph(image_source, h_glyph) plot.add_glyph(image_source, k_glyph) plot.add_glyph(image_source, l_glyph) plot.add_glyph(image_source, gamma_glyph) plot.add_glyph(image_source, nu_glyph) - plot.add_glyph(image_source, omega_glyph) image_glyph = Image(image="image", x="x", y="y", dw="dw", dh="dh") plot.add_glyph(image_source, image_glyph, name="image_glyph") @@ -228,7 +224,6 @@ def create(): ("intensity", "@image"), ("gamma", "@gamma"), ("nu", "@nu"), - ("omega", "@omega"), ("h", "@h"), ("k", "@k"), ("l", "@l"), From 9783e5e7aa2b0187ae7595f3bbafb7ef6d61d79b Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Wed, 21 Oct 2020 14:00:09 +0200 Subject: [PATCH 105/116] Update param_study_moduls.py Should work with new dict keys --- pyzebra/param_study_moduls.py | 91 +++++++++++++++++++++++++++++------ 1 file changed, 76 insertions(+), 15 deletions(-) diff --git a/pyzebra/param_study_moduls.py b/pyzebra/param_study_moduls.py index ea9280a..c249139 100644 --- a/pyzebra/param_study_moduls.py +++ b/pyzebra/param_study_moduls.py @@ -1,11 +1,12 @@ -import matplotlib as mpl -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -from load_1Dnn import load_1D -from mpl_toolkits.mplot3d import Axes3D # dont delete, otherwise waterfall wont work - +from load_1D import load_1D from ccl_dict_operation import add_dict +import pandas as pd +from mpl_toolkits.mplot3d import Axes3D # dont delete, otherwise waterfall wont work +import matplotlib.pyplot as plt +import matplotlib as mpl +import numpy as np +import pickle +import scipy.io as sio def load_dats(filepath): @@ -17,6 +18,7 @@ def load_dats(filepath): file_list = list() with open(filepath, "r") as infile: col_names = next(infile).split(",") + col_names = [col_names[i].rstrip() for i in range(len(col_names))] for line in infile: if "END" in line: break @@ -24,8 +26,6 @@ def load_dats(filepath): elif isinstance(filepath, list): data_type = "list" file_list = filepath - - print(file_list) dict1 = {} for i in range(len(file_list)): if not dict1: @@ -38,12 +38,12 @@ def load_dats(filepath): dict1 = add_dict(dict1, load_1D(file_list[i][0])) else: dict1 = add_dict(dict1, load_1D(file_list[i])) - dict1["meas"][str("M" + str(i + 1))]["params"] = {} + dict1["meas"][i + 1]["params"] = {} if data_type == "txt": for x in range(len(col_names) - 1): - dict1["meas"][str("M" + str(i + 1))]["params"][col_names[x + 1]] = file_list[i][ - x + 1 - ] + dict1["meas"][i + 1]["params"][ + col_names[x + 1] + ] = file_list[i][x + 1] return dict1 @@ -55,7 +55,7 @@ def create_dataframe(dict1): # create dictionary to which we pull only wanted items before transforming it to pd.dataframe pull_dict = {} pull_dict["filenames"] = list() - for key in dict1["meas"]["M1"]["params"]: + for key in dict1["meas"][1]["params"]: pull_dict[key] = list() pull_dict["temperature"] = list() pull_dict["mag_field"] = list() @@ -67,7 +67,9 @@ def create_dataframe(dict1): # populate the dict for keys in dict1["meas"]: if "file_of_origin" in dict1["meas"][keys]: - pull_dict["filenames"].append(dict1["meas"][keys]["file_of_origin"].split("/")[-1]) + pull_dict["filenames"].append( + dict1["meas"][keys]["file_of_origin"].split("/")[-1] + ) else: pull_dict["filenames"].append(dict1["meta"]["original_filename"].split("/")[-1]) for key in dict1["meas"][keys]["params"]: @@ -143,3 +145,62 @@ def make_graph(data, sorting_parameter, style): plt.clim(color_matrix.mean(), color_matrix.max()) return fig +def save_dict(obj, name): + """ saves dictionary as pickle file in binary format + :arg obj - object to save + :arg name - name of the file + NOTE: path should be added later""" + with open(name + '.pkl', 'wb') as f: + pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL) + + +def load_dict(name): + """load dictionary from picle file + :arg name - name of the file to load + NOTE: expect the file in the same folder, path should be added later + :return dictionary""" + with open(name + '.pkl', 'rb') as f: + return pickle.load(f) +# pickle, mat, h5, txt, csv, json +def save_table(data, filetype, name, path=None): + print("Saving: ", filetype) + path = "" if path is None else path + if filetype == "pickle": + # to work with uncertanities, see uncertanity module + with open(path + name + '.pkl', 'wb') as f: + pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) + if filetype == "mat": + # matlab doesent allow some special character to be in var names, also cant start with + # numbers, in need, add some to the romove_character list + data["fit_area_nom"] = [data["fit_area"][i].n for i in range(len(data["fit_area"]))] + data["fit_area_err"] = [data["fit_area"][i].s for i in range(len(data["fit_area"]))] + data["int_area_nom"] = [data["int_area"][i].n for i in range(len(data["int_area"]))] + data["int_area_err"] = [data["int_area"][i].s for i in range(len(data["int_area"]))] + data = data.drop(columns=['fit_area', 'int_area']) + remove_characters = [" ", "[", "]", "{", "}", "(",")"] + for character in remove_characters: + data.columns = [data.columns[i].replace(character,"") for i in range(len(data.columns))] + sio.savemat((path + name + '.mat'), {name: col.values for name, col in data.items()}) + if filetype == "csv" or "txt": + data["fit_area_nom"] = [data["fit_area"][i].n for i in range(len(data["fit_area"]))] + data["fit_area_err"] = [data["fit_area"][i].s for i in range(len(data["fit_area"]))] + data["int_area_nom"] = [data["int_area"][i].n for i in range(len(data["int_area"]))] + data["int_area_err"] = [data["int_area"][i].s for i in range(len(data["int_area"]))] + data = data.drop(columns=['fit_area', 'int_area', 'om', 'Counts']) + if filetype == "csv": + data.to_csv(path + name + '.csv') + if filetype == "txt": + with open((path + name + '.txt'), 'w') as outfile: + data.to_string(outfile) + if filetype == "h5": + hdf = pd.HDFStore((path + name + ".h5")) + hdf.put("data", data) + hdf.close() + if filetype == "json": + data.to_json((path + name + ".json")) + + + + + + From 31335b398b46884e628b44981c7bbb75aeae002f Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Wed, 21 Oct 2020 14:31:23 +0200 Subject: [PATCH 106/116] Update load_1D.py Replaced the "1" to 1 in dat part --- pyzebra/load_1D.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 44dbcd5..3ef6220 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -200,7 +200,7 @@ def parse_1D(fileobj, data_type): data_cols['omega_angle'] = metadata['omega'] data_cols['number_of_measurements'] = len(data_cols['om']) data_cols['monitor'] = data_cols['Monitor1'][0] - measurements["1"] = dict(data_cols) + measurements[1] = dict(data_cols) else: print("Unknown file extention") From f79553c1ba4135acbd7f758d5071e57757b31d98 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Wed, 21 Oct 2020 14:46:03 +0200 Subject: [PATCH 107/116] Update load_1D.py Added angles from meta to each scan, try except to try to get hkl from title --- pyzebra/load_1D.py | 125 +++++++++++++++++++-------------------------- 1 file changed, 53 insertions(+), 72 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 3ef6220..1b632a5 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -57,40 +57,6 @@ META_VARS_FLOAT = ( ) META_UB_MATRIX = ("ub1j", "ub2j", "ub3j") -CCL_FIRST_LINE = ( - # the first element is `measurement_number`, which we don't save to metadata - ("h_index", float), - ("k_index", float), - ("l_index", float), -) - -CCL_FIRST_LINE_BI = ( - *CCL_FIRST_LINE, - ("twotheta_angle", float), - ("omega_angle", float), - ("chi_angle", float), - ("phi_angle", float), -) - -CCL_FIRST_LINE_NB = ( - *CCL_FIRST_LINE, - ("gamma_angle", float), - ("omega_angle", float), - ("nu_angle", float), - ("unkwn_angle", float), -) - -CCL_SECOND_LINE = ( - ("number_of_measurements", int), - ("angle_step", float), - ("monitor", float), - ("temperature", float), - ("mag_field", float), - ("date", str), - ("time", str), - ("scan_type", str), -) - def load_1D(filepath): """ @@ -133,44 +99,51 @@ def parse_1D(fileobj, data_type): measurements = {} if data_type == ".ccl": decimal = list() - - if metadata["zebra_mode"] == "bi": - ccl_first_line = CCL_FIRST_LINE_BI - elif metadata["zebra_mode"] == "nb": - ccl_first_line = CCL_FIRST_LINE_NB - ccl_second_line = CCL_SECOND_LINE - for line in fileobj: - d = {} - - # first line - measurement_number, *params = line.split() - for param, (param_name, param_type) in zip(params, ccl_first_line): - d[param_name] = param_type(param) - - decimal.append(bool(Decimal(d["h_index"]) % 1 == 0)) - decimal.append(bool(Decimal(d["k_index"]) % 1 == 0)) - decimal.append(bool(Decimal(d["l_index"]) % 1 == 0)) - - # second line - next_line = next(fileobj) - params = next_line.split() - for param, (param_name, param_type) in zip(params, ccl_second_line): - d[param_name] = param_type(param) - - d["om"] = np.linspace( - d["omega_angle"] - (d["number_of_measurements"] / 2) * d["angle_step"], - d["omega_angle"] + (d["number_of_measurements"] / 2) * d["angle_step"], - d["number_of_measurements"], - ) - - # subsequent lines with counts counts = [] - while len(counts) < d["number_of_measurements"]: - counts.extend(map(int, next(fileobj).split())) - d["Counts"] = counts + measurement_number = int(line.split()[0]) + d = {} + d["h_index"] = float(line.split()[1]) + decimal.append(bool(Decimal(d["h_index"]) % 1 == 0)) + d["k_index"] = float(line.split()[2]) + decimal.append(bool(Decimal(d["k_index"]) % 1 == 0)) + d["l_index"] = float(line.split()[3]) + decimal.append(bool(Decimal(d["l_index"]) % 1 == 0)) + if metadata["zebra_mode"] == "bi": + d["twotheta_angle"] = float(line.split()[4]) # gamma + d["omega_angle"] = float(line.split()[5]) # omega + d["chi_angle"] = float(line.split()[6]) # nu + d["phi_angle"] = float(line.split()[7]) # doesnt matter + elif metadata["zebra_mode"] == "nb": + d["gamma_angle"] = float(line.split()[4]) # gamma + d["omega_angle"] = float(line.split()[5]) # omega + d["nu_angle"] = float(line.split()[6]) # nu + d["unkwn_angle"] = float(line.split()[7]) - measurements[int(measurement_number)] = d + next_line = next(fileobj) + d["number_of_measurements"] = int(next_line.split()[0]) + d["angle_step"] = float(next_line.split()[1]) + d["monitor"] = float(next_line.split()[2]) + d["temperature"] = float(next_line.split()[3]) + d["mag_field"] = float(next_line.split()[4]) + d["date"] = str(next_line.split()[5]) + d["time"] = str(next_line.split()[6]) + d["scan_type"] = str(next_line.split()[7]) + for i in range( + int(int(next_line.split()[0]) / 10) + (int(next_line.split()[0]) % 10 > 0) + ): + fileline = next(fileobj).split() + numbers = [int(w) for w in fileline] + counts = counts + numbers + d["om"] = np.linspace( + float(line.split()[5]) + - (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), + float(line.split()[5]) + + (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), + int(next_line.split()[0]), + ) + d["Counts"] = counts + measurements[measurement_number] = d if all(decimal): metadata["indices"] = "hkl" @@ -192,15 +165,23 @@ def parse_1D(fileobj, data_type): for name, val in zip(col_names, line.split()): data_cols[name].append(float(val)) - data_cols['h_index'] = float(metadata['title'].split()[-3]) - data_cols['k_index'] = float(metadata['title'].split()[-2]) - data_cols['l_index'] = float(metadata['title'].split()[-1]) + try: + data_cols['h_index'] = float(metadata['title'].split()[-3]) + data_cols['k_index'] = float(metadata['title'].split()[-2]) + data_cols['l_index'] = float(metadata['title'].split()[-1]) + except (ValueError, IndexError): + print("seems hkl is not in title") data_cols['temperature'] = metadata['temp'] data_cols['mag_field'] = metadata['mf'] data_cols['omega_angle'] = metadata['omega'] data_cols['number_of_measurements'] = len(data_cols['om']) data_cols['monitor'] = data_cols['Monitor1'][0] measurements[1] = dict(data_cols) + # add angles to scan from meta + data_cols["twotheta_angle"] = metadata['2-theta'] + data_cols["chi_angle"] = metadata['chi'] + data_cols["phi_angle"] = metadata['phi'] + data_cols["nu_angle"] = metadata['nu'] else: print("Unknown file extention") From 70abb97cb67e7fbea49bdcda709ee02a31bb5689 Mon Sep 17 00:00:00 2001 From: JakHolzer <53743814+JakHolzer@users.noreply.github.com> Date: Wed, 21 Oct 2020 19:11:24 +0200 Subject: [PATCH 108/116] Update load_1D.py Correcting my stupidity --- pyzebra/load_1D.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 1b632a5..c221a53 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -176,12 +176,13 @@ def parse_1D(fileobj, data_type): data_cols['omega_angle'] = metadata['omega'] data_cols['number_of_measurements'] = len(data_cols['om']) data_cols['monitor'] = data_cols['Monitor1'][0] - measurements[1] = dict(data_cols) + # add angles to scan from meta data_cols["twotheta_angle"] = metadata['2-theta'] data_cols["chi_angle"] = metadata['chi'] data_cols["phi_angle"] = metadata['phi'] data_cols["nu_angle"] = metadata['nu'] + measurements[1] = dict(data_cols) else: print("Unknown file extention") From db785380baacd58a11780e94d854801c2680ce3a Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 22 Oct 2020 10:28:48 +0200 Subject: [PATCH 109/116] Revert "Update load_1D.py" This reverts commit f79553c1ba4135acbd7f758d5071e57757b31d98. --- pyzebra/load_1D.py | 118 +++++++++++++++++++++++++++------------------ 1 file changed, 71 insertions(+), 47 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index c221a53..7761f27 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -57,6 +57,40 @@ META_VARS_FLOAT = ( ) META_UB_MATRIX = ("ub1j", "ub2j", "ub3j") +CCL_FIRST_LINE = ( + # the first element is `measurement_number`, which we don't save to metadata + ("h_index", float), + ("k_index", float), + ("l_index", float), +) + +CCL_FIRST_LINE_BI = ( + *CCL_FIRST_LINE, + ("twotheta_angle", float), + ("omega_angle", float), + ("chi_angle", float), + ("phi_angle", float), +) + +CCL_FIRST_LINE_NB = ( + *CCL_FIRST_LINE, + ("gamma_angle", float), + ("omega_angle", float), + ("nu_angle", float), + ("unkwn_angle", float), +) + +CCL_SECOND_LINE = ( + ("number_of_measurements", int), + ("angle_step", float), + ("monitor", float), + ("temperature", float), + ("mag_field", float), + ("date", str), + ("time", str), + ("scan_type", str), +) + def load_1D(filepath): """ @@ -99,51 +133,44 @@ def parse_1D(fileobj, data_type): measurements = {} if data_type == ".ccl": decimal = list() - for line in fileobj: - counts = [] - measurement_number = int(line.split()[0]) - d = {} - d["h_index"] = float(line.split()[1]) - decimal.append(bool(Decimal(d["h_index"]) % 1 == 0)) - d["k_index"] = float(line.split()[2]) - decimal.append(bool(Decimal(d["k_index"]) % 1 == 0)) - d["l_index"] = float(line.split()[3]) - decimal.append(bool(Decimal(d["l_index"]) % 1 == 0)) - if metadata["zebra_mode"] == "bi": - d["twotheta_angle"] = float(line.split()[4]) # gamma - d["omega_angle"] = float(line.split()[5]) # omega - d["chi_angle"] = float(line.split()[6]) # nu - d["phi_angle"] = float(line.split()[7]) # doesnt matter - elif metadata["zebra_mode"] == "nb": - d["gamma_angle"] = float(line.split()[4]) # gamma - d["omega_angle"] = float(line.split()[5]) # omega - d["nu_angle"] = float(line.split()[6]) # nu - d["unkwn_angle"] = float(line.split()[7]) + if metadata["zebra_mode"] == "bi": + ccl_first_line = CCL_FIRST_LINE_BI + elif metadata["zebra_mode"] == "nb": + ccl_first_line = CCL_FIRST_LINE_NB + ccl_second_line = CCL_SECOND_LINE + + for line in fileobj: + d = {} + + # first line + measurement_number, *params = line.split() + for param, (param_name, param_type) in zip(params, ccl_first_line): + d[param_name] = param_type(param) + + decimal.append(bool(Decimal(d["h_index"]) % 1 == 0)) + decimal.append(bool(Decimal(d["k_index"]) % 1 == 0)) + decimal.append(bool(Decimal(d["l_index"]) % 1 == 0)) + + # second line next_line = next(fileobj) - d["number_of_measurements"] = int(next_line.split()[0]) - d["angle_step"] = float(next_line.split()[1]) - d["monitor"] = float(next_line.split()[2]) - d["temperature"] = float(next_line.split()[3]) - d["mag_field"] = float(next_line.split()[4]) - d["date"] = str(next_line.split()[5]) - d["time"] = str(next_line.split()[6]) - d["scan_type"] = str(next_line.split()[7]) - for i in range( - int(int(next_line.split()[0]) / 10) + (int(next_line.split()[0]) % 10 > 0) - ): - fileline = next(fileobj).split() - numbers = [int(w) for w in fileline] - counts = counts + numbers + params = next_line.split() + for param, (param_name, param_type) in zip(params, ccl_second_line): + d[param_name] = param_type(param) + d["om"] = np.linspace( - float(line.split()[5]) - - (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), - float(line.split()[5]) - + (int(next_line.split()[0]) / 2) * float(next_line.split()[1]), - int(next_line.split()[0]), + d["omega_angle"] - (d["number_of_measurements"] / 2) * d["angle_step"], + d["omega_angle"] + (d["number_of_measurements"] / 2) * d["angle_step"], + d["number_of_measurements"], ) + + # subsequent lines with counts + counts = [] + while len(counts) < d["number_of_measurements"]: + counts.extend(map(int, next(fileobj).split())) d["Counts"] = counts - measurements[measurement_number] = d + + measurements[int(measurement_number)] = d if all(decimal): metadata["indices"] = "hkl" @@ -165,18 +192,15 @@ def parse_1D(fileobj, data_type): for name, val in zip(col_names, line.split()): data_cols[name].append(float(val)) - try: - data_cols['h_index'] = float(metadata['title'].split()[-3]) - data_cols['k_index'] = float(metadata['title'].split()[-2]) - data_cols['l_index'] = float(metadata['title'].split()[-1]) - except (ValueError, IndexError): - print("seems hkl is not in title") + data_cols['h_index'] = float(metadata['title'].split()[-3]) + data_cols['k_index'] = float(metadata['title'].split()[-2]) + data_cols['l_index'] = float(metadata['title'].split()[-1]) data_cols['temperature'] = metadata['temp'] data_cols['mag_field'] = metadata['mf'] data_cols['omega_angle'] = metadata['omega'] data_cols['number_of_measurements'] = len(data_cols['om']) data_cols['monitor'] = data_cols['Monitor1'][0] - + # add angles to scan from meta data_cols["twotheta_angle"] = metadata['2-theta'] data_cols["chi_angle"] = metadata['chi'] From 784058b54e36cbea0ceec030e59a9db75b7cd102 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 22 Oct 2020 10:36:04 +0200 Subject: [PATCH 110/116] Reapply changes --- pyzebra/load_1D.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index 7761f27..f208a89 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -192,20 +192,23 @@ def parse_1D(fileobj, data_type): for name, val in zip(col_names, line.split()): data_cols[name].append(float(val)) - data_cols['h_index'] = float(metadata['title'].split()[-3]) - data_cols['k_index'] = float(metadata['title'].split()[-2]) - data_cols['l_index'] = float(metadata['title'].split()[-1]) - data_cols['temperature'] = metadata['temp'] - data_cols['mag_field'] = metadata['mf'] - data_cols['omega_angle'] = metadata['omega'] - data_cols['number_of_measurements'] = len(data_cols['om']) - data_cols['monitor'] = data_cols['Monitor1'][0] + try: + data_cols["h_index"] = float(metadata["title"].split()[-3]) + data_cols["k_index"] = float(metadata["title"].split()[-2]) + data_cols["l_index"] = float(metadata["title"].split()[-1]) + except (ValueError, IndexError): + print("seems hkl is not in title") + + data_cols["temperature"] = metadata["temp"] + data_cols["mag_field"] = metadata["mf"] + data_cols["omega_angle"] = metadata["omega"] + data_cols["number_of_measurements"] = len(data_cols["om"]) + data_cols["monitor"] = data_cols["Monitor1"][0] + data_cols["twotheta_angle"] = metadata["2-theta"] + data_cols["chi_angle"] = metadata["chi"] + data_cols["phi_angle"] = metadata["phi"] + data_cols["nu_angle"] = metadata["nu"] - # add angles to scan from meta - data_cols["twotheta_angle"] = metadata['2-theta'] - data_cols["chi_angle"] = metadata['chi'] - data_cols["phi_angle"] = metadata['phi'] - data_cols["nu_angle"] = metadata['nu'] measurements[1] = dict(data_cols) else: From 0c38f9b2d74023bef4d3d906b82c2e38f5eab02c Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 22 Oct 2020 11:02:00 +0200 Subject: [PATCH 111/116] Display std dev for counts --- pyzebra/app/panel_ccl_integrate.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index c9e840d..0d75d8f 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -21,6 +21,7 @@ from bokeh.models import ( Panel, Plot, RadioButtonGroup, + Scatter, Select, Spacer, Span, @@ -29,6 +30,7 @@ from bokeh.models import ( TextAreaInput, TextInput, Toggle, + Whisker, ) import pyzebra @@ -114,7 +116,7 @@ def create(): y = meas["Counts"] x = meas["om"] - plot_line_source.data.update(x=x, y=y) + plot_scatter_source.data.update(x=x, y=y, y_upper=y + np.sqrt(y), y_lower=y - np.sqrt(y)) num_of_peaks = meas.get("num_of_peaks") if num_of_peaks is not None and num_of_peaks > 0: @@ -192,8 +194,9 @@ def create(): plot.add_layout(Grid(dimension=0, ticker=BasicTicker())) plot.add_layout(Grid(dimension=1, ticker=BasicTicker())) - plot_line_source = ColumnDataSource(dict(x=[0], y=[0])) - plot.add_glyph(plot_line_source, Line(x="x", y="y", line_color="steelblue")) + plot_scatter_source = ColumnDataSource(dict(x=[0], y=[0], y_upper=[0], y_lower=[0])) + plot.add_glyph(plot_scatter_source, Scatter(x="x", y="y", line_color="steelblue")) + plot.add_layout(Whisker(source=plot_scatter_source, base="x", upper="y_upper", lower="y_lower")) plot_line_smooth_source = ColumnDataSource(dict(x=[0], y=[0])) plot.add_glyph( From 44e461f1806508495989d190c522f5dc1e53a34c Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 22 Oct 2020 11:29:32 +0200 Subject: [PATCH 112/116] Make found peaks more visible --- pyzebra/app/panel_ccl_integrate.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 0d75d8f..298ec71 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -6,9 +6,9 @@ import tempfile import numpy as np from bokeh.layouts import column, row from bokeh.models import ( + Asterisk, BasicTicker, Button, - Circle, ColumnDataSource, CustomJS, DataRange1d, @@ -126,11 +126,11 @@ def create(): else: peak_pos_textinput.value = str([meas["om"][ind] for ind in peak_indexes]) - plot_circle_source.data.update(x=meas["om"][peak_indexes], y=meas["peak_heights"]) + plot_peak_source.data.update(x=meas["om"][peak_indexes], y=meas["peak_heights"]) plot_line_smooth_source.data.update(x=x, y=meas["smooth_peaks"]) else: peak_pos_textinput.value = None - plot_circle_source.data.update(x=[], y=[]) + plot_peak_source.data.update(x=[], y=[]) plot_line_smooth_source.data.update(x=[], y=[]) peak_pos_textinput_lock = False @@ -209,8 +209,8 @@ def create(): plot_bkg_source = ColumnDataSource(dict(x=[0], y=[0])) plot.add_glyph(plot_bkg_source, Line(x="x", y="y", line_color="green", line_dash="dashed")) - plot_circle_source = ColumnDataSource(dict(x=[], y=[])) - plot.add_glyph(plot_circle_source, Circle(x="x", y="y")) + plot_peak_source = ColumnDataSource(dict(x=[], y=[])) + plot.add_glyph(plot_peak_source, Asterisk(x="x", y="y", size=10, line_color="red")) numfit_min_span = Span(location=None, dimension="height", line_dash="dashed") plot.add_layout(numfit_min_span) From b2047585231c27da7a56012ddab9946a3df33b4f Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 22 Oct 2020 11:43:52 +0200 Subject: [PATCH 113/116] Add hkl indices to scan table --- pyzebra/app/panel_ccl_integrate.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 298ec71..38464c0 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -78,8 +78,12 @@ def create(): det_data = pyzebra.parse_1D(file, ext) meas_list = list(det_data["meas"].keys()) + hkl = [ + f'{int(m["h_index"])} {int(m["k_index"])} {int(m["l_index"])}' + for m in det_data["meas"].values() + ] meas_table_source.data.update( - measurement=meas_list, peaks=[0] * len(meas_list), fit=[0] * len(meas_list) + measurement=meas_list, hkl=hkl, peaks=[0] * len(meas_list), fit=[0] * len(meas_list) ) meas_table_source.selected.indices = [] meas_table_source.selected.indices = [0] @@ -94,8 +98,12 @@ def create(): det_data = pyzebra.parse_1D(file, ext) meas_list = list(det_data["meas"].keys()) + hkl = [ + f'{int(m["h_index"])} {int(m["k_index"])} {int(m["l_index"])}' + for m in det_data["meas"].values() + ] meas_table_source.data.update( - measurement=meas_list, peaks=[0] * len(meas_list), fit=[0] * len(meas_list) + measurement=meas_list, hkl=hkl, peaks=[0] * len(meas_list), fit=[0] * len(meas_list) ) meas_table_source.selected.indices = [] meas_table_source.selected.indices = [0] @@ -223,15 +231,16 @@ def create(): if new: _update_plot(meas_table_source.data["measurement"][new[-1]]) - meas_table_source = ColumnDataSource(dict(measurement=[], peaks=[], fit=[])) + meas_table_source = ColumnDataSource(dict(measurement=[], hkl=[], peaks=[], fit=[])) meas_table = DataTable( source=meas_table_source, columns=[ TableColumn(field="measurement", title="Meas"), + TableColumn(field="hkl", title="hkl"), TableColumn(field="peaks", title="Peaks"), TableColumn(field="fit", title="Fit"), ], - width=150, + width=200, index_position=None, ) From 199f8d9f615bc3e37d4af84e66367e57d4571013 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Thu, 22 Oct 2020 12:03:19 +0200 Subject: [PATCH 114/116] Use 'scan' instead of 'meas' for array of counts --- pyzebra/app/panel_ccl_integrate.py | 120 +++++++++--------- pyzebra/ccl_dict_operation.py | 196 ++++++++++++++--------------- pyzebra/ccl_findpeaks.py | 18 +-- pyzebra/comm_export.py | 44 +++---- pyzebra/fit2.py | 42 +++---- pyzebra/load_1D.py | 16 +-- pyzebra/param_study_moduls.py | 64 +++++----- 7 files changed, 248 insertions(+), 252 deletions(-) diff --git a/pyzebra/app/panel_ccl_integrate.py b/pyzebra/app/panel_ccl_integrate.py index 38464c0..16e5be5 100644 --- a/pyzebra/app/panel_ccl_integrate.py +++ b/pyzebra/app/panel_ccl_integrate.py @@ -77,16 +77,16 @@ def create(): _, ext = os.path.splitext(new) det_data = pyzebra.parse_1D(file, ext) - meas_list = list(det_data["meas"].keys()) + scan_list = list(det_data["scan"].keys()) hkl = [ f'{int(m["h_index"])} {int(m["k_index"])} {int(m["l_index"])}' - for m in det_data["meas"].values() + for m in det_data["scan"].values() ] - meas_table_source.data.update( - measurement=meas_list, hkl=hkl, peaks=[0] * len(meas_list), fit=[0] * len(meas_list) + scan_table_source.data.update( + scan=scan_list, hkl=hkl, peaks=[0] * len(scan_list), fit=[0] * len(scan_list) ) - meas_table_source.selected.indices = [] - meas_table_source.selected.indices = [0] + scan_table_source.selected.indices = [] + scan_table_source.selected.indices = [0] ccl_file_select = Select(title="Available .ccl files") ccl_file_select.on_change("value", ccl_file_select_callback) @@ -97,45 +97,45 @@ def create(): _, ext = os.path.splitext(upload_button.filename) det_data = pyzebra.parse_1D(file, ext) - meas_list = list(det_data["meas"].keys()) + scan_list = list(det_data["scan"].keys()) hkl = [ f'{int(m["h_index"])} {int(m["k_index"])} {int(m["l_index"])}' - for m in det_data["meas"].values() + for m in det_data["scan"].values() ] - meas_table_source.data.update( - measurement=meas_list, hkl=hkl, peaks=[0] * len(meas_list), fit=[0] * len(meas_list) + scan_table_source.data.update( + scan=scan_list, hkl=hkl, peaks=[0] * len(scan_list), fit=[0] * len(scan_list) ) - meas_table_source.selected.indices = [] - meas_table_source.selected.indices = [0] + scan_table_source.selected.indices = [] + scan_table_source.selected.indices = [0] upload_button = FileInput(accept=".ccl") upload_button.on_change("value", upload_button_callback) def _update_table(): - num_of_peaks = [meas.get("num_of_peaks", 0) for meas in det_data["meas"].values()] - fit_ok = [(1 if "fit" in meas else 0) for meas in det_data["meas"].values()] - meas_table_source.data.update(peaks=num_of_peaks, fit=fit_ok) + num_of_peaks = [scan.get("num_of_peaks", 0) for scan in det_data["scan"].values()] + fit_ok = [(1 if "fit" in scan else 0) for scan in det_data["scan"].values()] + scan_table_source.data.update(peaks=num_of_peaks, fit=fit_ok) def _update_plot(ind): nonlocal peak_pos_textinput_lock peak_pos_textinput_lock = True - meas = det_data["meas"][ind] - y = meas["Counts"] - x = meas["om"] + scan = det_data["scan"][ind] + y = scan["Counts"] + x = scan["om"] plot_scatter_source.data.update(x=x, y=y, y_upper=y + np.sqrt(y), y_lower=y - np.sqrt(y)) - num_of_peaks = meas.get("num_of_peaks") + num_of_peaks = scan.get("num_of_peaks") if num_of_peaks is not None and num_of_peaks > 0: - peak_indexes = meas["peak_indexes"] + peak_indexes = scan["peak_indexes"] if len(peak_indexes) == 1: - peak_pos_textinput.value = str(meas["om"][peak_indexes[0]]) + peak_pos_textinput.value = str(scan["om"][peak_indexes[0]]) else: - peak_pos_textinput.value = str([meas["om"][ind] for ind in peak_indexes]) + peak_pos_textinput.value = str([scan["om"][ind] for ind in peak_indexes]) - plot_peak_source.data.update(x=meas["om"][peak_indexes], y=meas["peak_heights"]) - plot_line_smooth_source.data.update(x=x, y=meas["smooth_peaks"]) + plot_peak_source.data.update(x=scan["om"][peak_indexes], y=scan["peak_heights"]) + plot_line_smooth_source.data.update(x=x, y=scan["smooth_peaks"]) else: peak_pos_textinput.value = None plot_peak_source.data.update(x=[], y=[]) @@ -143,10 +143,10 @@ def create(): peak_pos_textinput_lock = False - fit = meas.get("fit") + fit = scan.get("fit") if fit is not None: - plot_gauss_source.data.update(x=x, y=meas["fit"]["comps"]["gaussian"]) - plot_bkg_source.data.update(x=x, y=meas["fit"]["comps"]["background"]) + plot_gauss_source.data.update(x=x, y=scan["fit"]["comps"]["gaussian"]) + plot_bkg_source.data.update(x=x, y=scan["fit"]["comps"]["background"]) params = fit["result"].params fit_output_textinput.value = ( "%s \n" @@ -226,16 +226,16 @@ def create(): numfit_max_span = Span(location=None, dimension="height", line_dash="dashed") plot.add_layout(numfit_max_span) - # Measurement select - def meas_table_callback(_attr, _old, new): + # Scan select + def scan_table_callback(_attr, _old, new): if new: - _update_plot(meas_table_source.data["measurement"][new[-1]]) + _update_plot(scan_table_source.data["scan"][new[-1]]) - meas_table_source = ColumnDataSource(dict(measurement=[], hkl=[], peaks=[], fit=[])) - meas_table = DataTable( - source=meas_table_source, + scan_table_source = ColumnDataSource(dict(scan=[], hkl=[], peaks=[], fit=[])) + scan_table = DataTable( + source=scan_table_source, columns=[ - TableColumn(field="measurement", title="Meas"), + TableColumn(field="scan", title="scan"), TableColumn(field="hkl", title="hkl"), TableColumn(field="peaks", title="Peaks"), TableColumn(field="fit", title="Fit"), @@ -244,20 +244,20 @@ def create(): index_position=None, ) - meas_table_source.selected.on_change("indices", meas_table_callback) + scan_table_source.selected.on_change("indices", scan_table_callback) def peak_pos_textinput_callback(_attr, _old, new): if new is not None and not peak_pos_textinput_lock: - sel_ind = meas_table_source.selected.indices[-1] - meas_name = meas_table_source.data["measurement"][sel_ind] - meas = det_data["meas"][meas_name] + sel_ind = scan_table_source.selected.indices[-1] + scan_name = scan_table_source.data["scan"][sel_ind] + scan = det_data["scan"][scan_name] - meas["num_of_peaks"] = 1 - peak_ind = (np.abs(meas["om"] - float(new))).argmin() - meas["peak_indexes"] = np.array([peak_ind], dtype=np.int64) - meas["peak_heights"] = np.array([meas["smooth_peaks"][peak_ind]]) + scan["num_of_peaks"] = 1 + peak_ind = (np.abs(scan["om"] - float(new))).argmin() + scan["peak_indexes"] = np.array([peak_ind], dtype=np.int64) + scan["peak_heights"] = np.array([scan["smooth_peaks"][peak_ind]]) _update_table() - _update_plot(meas_name) + _update_plot(scan_name) peak_pos_textinput = TextInput(title="Peak position:", default_size=145) peak_pos_textinput.on_change("value", peak_pos_textinput_callback) @@ -323,9 +323,9 @@ def create(): fit_output_textinput = TextAreaInput(title="Fit results:", width=450, height=400) def peakfind_all_button_callback(): - for meas in det_data["meas"].values(): + for scan in det_data["scan"].values(): pyzebra.ccl_findpeaks( - meas, + scan, int_threshold=peak_int_ratio_spinner.value, prominence=peak_prominence_spinner.value, smooth=smooth_toggle.active, @@ -335,17 +335,17 @@ def create(): _update_table() - sel_ind = meas_table_source.selected.indices[-1] - _update_plot(meas_table_source.data["measurement"][sel_ind]) + sel_ind = scan_table_source.selected.indices[-1] + _update_plot(scan_table_source.data["scan"][sel_ind]) peakfind_all_button = Button(label="Peak Find All", button_type="primary", default_size=145) peakfind_all_button.on_click(peakfind_all_button_callback) def peakfind_button_callback(): - sel_ind = meas_table_source.selected.indices[-1] - meas = meas_table_source.data["measurement"][sel_ind] + sel_ind = scan_table_source.selected.indices[-1] + scan = scan_table_source.data["scan"][sel_ind] pyzebra.ccl_findpeaks( - det_data["meas"][meas], + det_data["scan"][scan], int_threshold=peak_int_ratio_spinner.value, prominence=peak_prominence_spinner.value, smooth=smooth_toggle.active, @@ -354,15 +354,15 @@ def create(): ) _update_table() - _update_plot(meas) + _update_plot(scan) peakfind_button = Button(label="Peak Find Current", default_size=145) peakfind_button.on_click(peakfind_button_callback) def fit_all_button_callback(): - for meas in det_data["meas"].values(): + for scan in det_data["scan"].values(): pyzebra.fitccl( - meas, + scan, guess=[ centre_guess.value, sigma_guess.value, @@ -395,19 +395,19 @@ def create(): numfit_max=integ_to.value, ) - sel_ind = meas_table_source.selected.indices[-1] - _update_plot(meas_table_source.data["measurement"][sel_ind]) + sel_ind = scan_table_source.selected.indices[-1] + _update_plot(scan_table_source.data["scan"][sel_ind]) _update_table() fit_all_button = Button(label="Fit All", button_type="primary", default_size=145) fit_all_button.on_click(fit_all_button_callback) def fit_button_callback(): - sel_ind = meas_table_source.selected.indices[-1] - meas = meas_table_source.data["measurement"][sel_ind] + sel_ind = scan_table_source.selected.indices[-1] + scan = scan_table_source.data["scan"][sel_ind] pyzebra.fitccl( - det_data["meas"][meas], + det_data["scan"][scan], guess=[ centre_guess.value, sigma_guess.value, @@ -440,7 +440,7 @@ def create(): numfit_max=integ_to.value, ) - _update_plot(meas) + _update_plot(scan) _update_table() fit_button = Button(label="Fit Current", default_size=145) @@ -541,7 +541,7 @@ def create(): tab_layout = column( row(proposal_textinput, ccl_file_select), row(column(Spacer(height=5), upload_div), upload_button), - row(meas_table, plot, Spacer(width=30), fit_output_textinput, export_layout), + row(scan_table, plot, Spacer(width=30), fit_output_textinput, export_layout), row(findpeak_controls, Spacer(width=30), fitpeak_controls), ) diff --git a/pyzebra/ccl_dict_operation.py b/pyzebra/ccl_dict_operation.py index 42b4766..7d3b40d 100644 --- a/pyzebra/ccl_dict_operation.py +++ b/pyzebra/ccl_dict_operation.py @@ -12,17 +12,17 @@ def add_dict(dict1, dict2): :return dict1 : combined dictionary Note: dict1 must be made from ccl, otherwise we would have to change the structure of loaded dat file""" - max_measurement_dict1 = max([int(str(keys)[1:]) for keys in dict1["meas"]]) + max_measurement_dict1 = max([int(str(keys)[1:]) for keys in dict1["scan"]]) if dict2["meta"]["data_type"] == ".ccl": new_filenames = [ "M" + str(x + max_measurement_dict1) - for x in [int(str(keys)[1:]) for keys in dict2["meas"]] + for x in [int(str(keys)[1:]) for keys in dict2["scan"]] ] new_meta_name = "meta" + str(dict2["meta"]["original_filename"]) if new_meta_name not in dict1: - for keys, name in zip(dict2["meas"], new_filenames): - dict2["meas"][keys]["file_of_origin"] = str(dict2["meta"]["original_filename"]) - dict1["meas"][name] = dict2["meas"][keys] + for keys, name in zip(dict2["scan"], new_filenames): + dict2["scan"][keys]["file_of_origin"] = str(dict2["meta"]["original_filename"]) + dict1["scan"][name] = dict2["scan"][keys] dict1[new_meta_name] = dict2["meta"] @@ -40,14 +40,14 @@ def add_dict(dict1, dict2): d["h_index"] = float(hkl.split()[-3]) d["k_index"] = float(hkl.split()[-2]) d["l_index"] = float(hkl.split()[-1]) - d["number_of_measurements"] = len(dict2["meas"]["NP"]) - d["om"] = dict2["meas"]["om"] - d["Counts"] = dict2["meas"]["Counts"] - d["monitor"] = dict2["meas"]["Monitor1"][0] + d["number_of_measurements"] = len(dict2["scan"]["NP"]) + d["om"] = dict2["scan"]["om"] + d["Counts"] = dict2["scan"]["Counts"] + d["monitor"] = dict2["scan"]["Monitor1"][0] d["temperature"] = dict2["meta"]["temp"] d["mag_field"] = dict2["meta"]["mf"] d["omega_angle"] = dict2["meta"]["omega"] - dict1["meas"][new_name] = d + dict1["scan"][new_name] = d print(hkl.split()) for keys in d: print(keys) @@ -80,18 +80,18 @@ def scan_dict(dict): """ d = {} - for i in dict["meas"]: - for j in dict["meas"]: - if dict["meas"][str(i)] != dict["meas"][str(j)]: + for i in dict["scan"]: + for j in dict["scan"]: + if dict["scan"][str(i)] != dict["scan"][str(j)]: itup = ( - dict["meas"][str(i)]["h_index"], - dict["meas"][str(i)]["k_index"], - dict["meas"][str(i)]["l_index"], + dict["scan"][str(i)]["h_index"], + dict["scan"][str(i)]["k_index"], + dict["scan"][str(i)]["l_index"], ) jtup = ( - dict["meas"][str(j)]["h_index"], - dict["meas"][str(j)]["k_index"], - dict["meas"][str(j)]["l_index"], + dict["scan"][str(j)]["h_index"], + dict["scan"][str(j)]["k_index"], + dict["scan"][str(j)]["l_index"], ) if itup != jtup: pass @@ -109,52 +109,52 @@ def scan_dict(dict): def compare_hkl(dict1, dict2): """Compares two dictionaries based on hkl indexes and return dictionary with str(h k l) as - key and tuple with keys to same measurement in dict1 and dict2 + key and tuple with keys to same scan in dict1 and dict2 :arg dict1 : first dictionary :arg dict2 : second dictionary :return d : dict with matches - example of one key: '0.0 0.0 -1.0 : ('M1', 'M9')' meaning that 001 hkl measurement is M1 in + example of one key: '0.0 0.0 -1.0 : ('M1', 'M9')' meaning that 001 hkl scan is M1 in first dict and M9 in second""" d = {} dupl = 0 - for keys in dict1["meas"]: - for key in dict2["meas"]: + for keys in dict1["scan"]: + for key in dict2["scan"]: if ( - dict1["meas"][str(keys)]["h_index"] == dict2["meas"][str(key)]["h_index"] - and dict1["meas"][str(keys)]["k_index"] == dict2["meas"][str(key)]["k_index"] - and dict1["meas"][str(keys)]["l_index"] == dict2["meas"][str(key)]["l_index"] + dict1["scan"][str(keys)]["h_index"] == dict2["scan"][str(key)]["h_index"] + and dict1["scan"][str(keys)]["k_index"] == dict2["scan"][str(key)]["k_index"] + and dict1["scan"][str(keys)]["l_index"] == dict2["scan"][str(key)]["l_index"] ): if ( str( ( - str(dict1["meas"][str(keys)]["h_index"]) + str(dict1["scan"][str(keys)]["h_index"]) + " " - + str(dict1["meas"][str(keys)]["k_index"]) + + str(dict1["scan"][str(keys)]["k_index"]) + " " - + str(dict1["meas"][str(keys)]["l_index"]) + + str(dict1["scan"][str(keys)]["l_index"]) ) ) not in d ): d[ str( - str(dict1["meas"][str(keys)]["h_index"]) + str(dict1["scan"][str(keys)]["h_index"]) + " " - + str(dict1["meas"][str(keys)]["k_index"]) + + str(dict1["scan"][str(keys)]["k_index"]) + " " - + str(dict1["meas"][str(keys)]["l_index"]) + + str(dict1["scan"][str(keys)]["l_index"]) ) ] = (str(keys), str(key)) else: dupl = dupl + 1 d[ str( - str(dict1["meas"][str(keys)]["h_index"]) + str(dict1["scan"][str(keys)]["h_index"]) + " " - + str(dict1["meas"][str(keys)]["k_index"]) + + str(dict1["scan"][str(keys)]["k_index"]) + " " - + str(dict1["meas"][str(keys)]["l_index"]) + + str(dict1["scan"][str(keys)]["l_index"]) + "_dupl" + str(dupl) ) @@ -176,16 +176,16 @@ def create_tuples(x, y, y_err): def normalize(dict, key, monitor): - """Normalizes the measurement to monitor, checks if sigma exists, otherwise creates it + """Normalizes the scan to monitor, checks if sigma exists, otherwise creates it :arg dict : dictionary to from which to tkae the scan :arg key : which scan to normalize from dict1 :arg monitor : final monitor :return counts - normalized counts :return sigma - normalized sigma""" - counts = np.array(dict["meas"][key]["Counts"]) - sigma = np.sqrt(counts) if "sigma" not in dict["meas"][key] else dict["meas"][key]["sigma"] - monitor_ratio = monitor / dict["meas"][key]["monitor"] + counts = np.array(dict["scan"][key]["Counts"]) + sigma = np.sqrt(counts) if "sigma" not in dict["scan"][key] else dict["scan"][key]["sigma"] + monitor_ratio = monitor / dict["scan"][key]["monitor"] scaled_counts = counts * monitor_ratio scaled_sigma = np.array(sigma) * monitor_ratio @@ -195,19 +195,19 @@ def normalize(dict, key, monitor): def merge(dict1, dict2, keys, auto=True, monitor=100000): """merges the two tuples and sorts them, if om value is same, Counts value is average averaging is propagated into sigma if dict1 == dict2, key[1] is deleted after merging - :arg dict1 : dictionary to which measurement will be merged - :arg dict2 : dictionary from which measurement will be merged + :arg dict1 : dictionary to which scan will be merged + :arg dict2 : dictionary from which scan will be merged :arg keys : tuple with key to dict1 and dict2 :arg auto : if true, when monitors are same, does not change it, if flase, takes monitor always :arg monitor : final monitor after merging note: dict1 and dict2 can be same dict :return dict1 with merged scan""" if auto: - if dict1["meas"][keys[0]]["monitor"] == dict2["meas"][keys[1]]["monitor"]: - monitor = dict1["meas"][keys[0]]["monitor"] + if dict1["scan"][keys[0]]["monitor"] == dict2["scan"][keys[1]]["monitor"]: + monitor = dict1["scan"][keys[0]]["monitor"] # load om and Counts - x1, x2 = dict1["meas"][keys[0]]["om"], dict2["meas"][keys[1]]["om"] + x1, x2 = dict1["scan"][keys[0]]["om"], dict2["scan"][keys[1]]["om"] cor_y1, y_err1 = normalize(dict1, keys[0], monitor=monitor) cor_y2, y_err2 = normalize(dict2, keys[1], monitor=monitor) # creates touples (om, Counts, sigma) for sorting and further processing @@ -237,40 +237,40 @@ def merge(dict1, dict2, keys, auto=True, monitor=100000): continue if dict1 == dict2: - del dict1["meas"][keys[1]] + del dict1["scan"][keys[1]] note = ( - f"This measurement was merged with measurement {keys[1]} from " + f"This scan was merged with scan {keys[1]} from " f'file {dict2["meta"]["original_filename"]} \n' ) - if "notes" not in dict1["meas"][str(keys[0])]: - dict1["meas"][str(keys[0])]["notes"] = note + if "notes" not in dict1["scan"][str(keys[0])]: + dict1["scan"][str(keys[0])]["notes"] = note else: - dict1["meas"][str(keys[0])]["notes"] += note + dict1["scan"][str(keys[0])]["notes"] += note - dict1["meas"][keys[0]]["om"] = om - dict1["meas"][keys[0]]["Counts"] = Counts - dict1["meas"][keys[0]]["sigma"] = sigma - dict1["meas"][keys[0]]["monitor"] = monitor + dict1["scan"][keys[0]]["om"] = om + dict1["scan"][keys[0]]["Counts"] = Counts + dict1["scan"][keys[0]]["sigma"] = sigma + dict1["scan"][keys[0]]["monitor"] = monitor print("merging done") return dict1 def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): - """Substracts two measurement (measurement key2 from dict2 from measurent key1 in dict1), expects om to be same - :arg dict1 : dictionary to which measurement will be merged - :arg dict2 : dictionary from which measurement will be merged + """Substracts two scan (scan key2 from dict2 from measurent key1 in dict1), expects om to be same + :arg dict1 : dictionary to which scan will be merged + :arg dict2 : dictionary from which scan will be merged :arg keys : tuple with key to dict1 and dict2 :arg auto : if true, when monitors are same, does not change it, if flase, takes monitor always :arg monitor : final monitor after merging :returns d : dict1 with substracted Counts from dict2 and sigma that comes from the substraction""" - if len(dict1["meas"][keys[0]]["om"]) != len(dict2["meas"][keys[1]]["om"]): + if len(dict1["scan"][keys[0]]["om"]) != len(dict2["scan"][keys[1]]["om"]): raise ValueError("Omegas have different lengths, cannot be substracted") if auto: - if dict1["meas"][keys[0]]["monitor"] == dict2["meas"][keys[1]]["monitor"]: - monitor = dict1["meas"][keys[0]]["monitor"] + if dict1["scan"][keys[0]]["monitor"] == dict2["scan"][keys[1]]["monitor"]: + monitor = dict1["scan"][keys[0]]["monitor"] cor_y1, y_err1 = normalize(dict1, keys[0], monitor=monitor) cor_y2, y_err2 = normalize(dict2, keys[1], monitor=monitor) @@ -288,21 +288,21 @@ def substract_measurement(dict1, dict2, keys, auto=True, monitor=100000): if len([num for num in res_nom if num < 0]) >= 0.3 * len(res_nom): print( - f"Warning! percentage of negative numbers in measurement subsracted {keys[0]} is " + f"Warning! percentage of negative numbers in scan subsracted {keys[0]} is " f"{len([num for num in res_nom if num < 0]) / len(res_nom)}" ) - dict1["meas"][str(keys[0])]["Counts"] = res_nom - dict1["meas"][str(keys[0])]["sigma"] = res_err - dict1["meas"][str(keys[0])]["monitor"] = monitor + dict1["scan"][str(keys[0])]["Counts"] = res_nom + dict1["scan"][str(keys[0])]["sigma"] = res_err + dict1["scan"][str(keys[0])]["monitor"] = monitor note = ( - f'Measurement {keys[1]} from file {dict2["meta"]["original_filename"]} ' - f"was substracted from this measurement \n" + f'Scan {keys[1]} from file {dict2["meta"]["original_filename"]} ' + f"was substracted from this scan \n" ) - if "notes" not in dict1["meas"][str(keys[0])]: - dict1["meas"][str(keys[0])]["notes"] = note + if "notes" not in dict1["scan"][str(keys[0])]: + dict1["scan"][str(keys[0])]["notes"] = note else: - dict1["meas"][str(keys[0])]["notes"] += note + dict1["scan"][str(keys[0])]["notes"] += note return dict1 @@ -311,7 +311,7 @@ def compare_dict(dict1, dict2): :arg dict1 : dictionary 1 (ccl) :arg dict2 : dictionary 2 (ccl) :returns warning : dictionary with keys from primary files (if they differ) with - information of how many measurement differ and which ones differ + information of how many scan differ and which ones differ :returns report_string string comparing all different values respecively of measurements""" if dict1["meta"]["data_type"] != dict2["meta"]["data_type"]: @@ -371,48 +371,48 @@ def compare_dict(dict1, dict2): # compare Measurements S.append( "Number of measurements in %s = %s \n" - % (dict1["meta"]["original_filename"], len(dict1["meas"])) + % (dict1["meta"]["original_filename"], len(dict1["scan"])) ) S.append( "Number of measurements in %s = %s \n" - % (dict2["meta"]["original_filename"], len(dict2["meas"])) + % (dict2["meta"]["original_filename"], len(dict2["scan"])) ) S.append("Different values in Measurements:\n") select_set = ["om", "Counts", "sigma"] exlude_set = ["time", "Counts", "date", "notes"] for keys1 in comp: - for key2 in dict1["meas"][str(comp[str(keys1)][0])]: + for key2 in dict1["scan"][str(comp[str(keys1)][0])]: if key2 in exlude_set: continue if key2 not in select_set: try: if ( - dict1["meas"][comp[str(keys1)][0]][str(key2)] - != dict2["meas"][str(comp[str(keys1)][1])][str(key2)] + dict1["scan"][comp[str(keys1)][0]][str(key2)] + != dict2["scan"][str(comp[str(keys1)][1])][str(key2)] ): S.append( - "Measurement value " + "Scan value " "%s" ", with hkl %s differs in meausrements %s and %s \n" % (key2, keys1, comp[str(keys1)][0], comp[str(keys1)][1]) ) S.append( " dict1: %s \n" - % str(dict1["meas"][comp[str(keys1)][0]][str(key2)]) + % str(dict1["scan"][comp[str(keys1)][0]][str(key2)]) ) S.append( " dict2: %s \n" - % str(dict2["meas"][comp[str(keys1)][1]][str(key2)]) + % str(dict2["scan"][comp[str(keys1)][1]][str(key2)]) ) if key2 not in conflicts: conflicts[key2] = {} conflicts[key2]["amount"] = 1 - conflicts[key2]["meas"] = str(comp[str(keys1)]) + conflicts[key2]["scan"] = str(comp[str(keys1)]) else: conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 - conflicts[key2]["meas"] = ( - conflicts[key2]["meas"] + " " + (str(comp[str(keys1)])) + conflicts[key2]["scan"] = ( + conflicts[key2]["scan"] + " " + (str(comp[str(keys1)])) ) except KeyError as e: print("Missing keys, some files were probably merged or substracted") @@ -420,11 +420,11 @@ def compare_dict(dict1, dict2): else: try: - comparison = list(dict1["meas"][comp[str(keys1)][0]][str(key2)]) == list( - dict2["meas"][comp[str(keys1)][1]][str(key2)] + comparison = list(dict1["scan"][comp[str(keys1)][0]][str(key2)]) == list( + dict2["scan"][comp[str(keys1)][1]][str(key2)] ) - if len(list(dict1["meas"][comp[str(keys1)][0]][str(key2)])) != len( - list(dict2["meas"][comp[str(keys1)][1]][str(key2)]) + if len(list(dict1["scan"][comp[str(keys1)][0]][str(key2)])) != len( + list(dict2["scan"][comp[str(keys1)][1]][str(key2)]) ): if str("different length of %s" % key2) not in warnings: warnings[str("different length of %s" % key2)] = list() @@ -437,27 +437,27 @@ def compare_dict(dict1, dict2): ) if not comparison: S.append( - "Measurement value " + "Scan value " "%s" - " differs in measurement %s and %s \n" + " differs in scan %s and %s \n" % (key2, comp[str(keys1)][0], comp[str(keys1)][1]) ) S.append( " dict1: %s \n" - % str(list(dict1["meas"][comp[str(keys1)][0]][str(key2)])) + % str(list(dict1["scan"][comp[str(keys1)][0]][str(key2)])) ) S.append( " dict2: %s \n" - % str(list(dict2["meas"][comp[str(keys1)][1]][str(key2)])) + % str(list(dict2["scan"][comp[str(keys1)][1]][str(key2)])) ) if key2 not in conflicts: conflicts[key2] = {} conflicts[key2]["amount"] = 1 - conflicts[key2]["meas"] = str(comp[str(keys1)]) + conflicts[key2]["scan"] = str(comp[str(keys1)]) else: conflicts[key2]["amount"] = conflicts[key2]["amount"] + 1 - conflicts[key2]["meas"] = ( - conflicts[key2]["meas"] + " " + (str(comp[str(keys1)])) + conflicts[key2]["scan"] = ( + conflicts[key2]["scan"] + " " + (str(comp[str(keys1)])) ) except KeyError as e: print("Missing keys, some files were probably merged or substracted") @@ -465,7 +465,7 @@ def compare_dict(dict1, dict2): for keys in conflicts: try: - conflicts[str(keys)]["meas"] = conflicts[str(keys)]["meas"].split(" ") + conflicts[str(keys)]["scan"] = conflicts[str(keys)]["scan"].split(" ") except: continue report_string = "".join(S) @@ -480,18 +480,18 @@ def guess_next(dict1, dict2, comp): if ( abs( ( - dict1["meas"][str(comp[keys][0])]["temperature"] - - dict2["meas"][str(comp[keys][1])]["temperature"] + dict1["scan"][str(comp[keys][0])]["temperature"] + - dict2["scan"][str(comp[keys][1])]["temperature"] ) - / dict2["meas"][str(comp[keys][1])]["temperature"] + / dict2["scan"][str(comp[keys][1])]["temperature"] ) < threshold and abs( ( - dict1["meas"][str(comp[keys][0])]["mag_field"] - - dict2["meas"][str(comp[keys][1])]["mag_field"] + dict1["scan"][str(comp[keys][0])]["mag_field"] + - dict2["scan"][str(comp[keys][1])]["mag_field"] ) - / dict2["meas"][str(comp[keys][1])]["mag_field"] + / dict2["scan"][str(comp[keys][1])]["mag_field"] ) < threshold ): diff --git a/pyzebra/ccl_findpeaks.py b/pyzebra/ccl_findpeaks.py index 6153d87..6e7d898 100644 --- a/pyzebra/ccl_findpeaks.py +++ b/pyzebra/ccl_findpeaks.py @@ -5,11 +5,11 @@ from scipy.signal import savgol_filter def ccl_findpeaks( - meas, int_threshold=0.8, prominence=50, smooth=False, window_size=7, poly_order=3 + scan, int_threshold=0.8, prominence=50, smooth=False, window_size=7, poly_order=3 ): - """function iterates through the dictionary created by load_cclv2 and locates peaks for each measurement - args: meas - a single measurement, + """function iterates through the dictionary created by load_cclv2 and locates peaks for each scan + args: scan - a single scan, int_threshold - fraction of threshold_intensity/max_intensity, must be positive num between 0 and 1 i.e. will only detect peaks above 75% of max intensity @@ -54,8 +54,8 @@ def ccl_findpeaks( prominence = 50 print("Invalid value for prominence, select positive number, new value set to:", prominence) - omega = meas["om"] - counts = np.array(meas["Counts"]) + omega = scan["om"] + counts = np.array(scan["Counts"]) if smooth: itp = interp1d(omega, counts, kind="linear") absintensity = [abs(number) for number in counts] @@ -69,7 +69,7 @@ def ccl_findpeaks( peaks, properties = sc.signal.find_peaks( smooth_peaks, height=int_threshold * max(smooth_peaks), prominence=prominence ) - meas["num_of_peaks"] = len(peaks) - meas["peak_indexes"] = peaks - meas["peak_heights"] = properties["peak_heights"] - meas["smooth_peaks"] = smooth_peaks # smoothed curve + scan["num_of_peaks"] = len(peaks) + scan["peak_indexes"] = peaks + scan["peak_heights"] = properties["peak_heights"] + scan["smooth_peaks"] = smooth_peaks # smoothed curve diff --git a/pyzebra/comm_export.py b/pyzebra/comm_export.py index 3ab2324..e178bb0 100644 --- a/pyzebra/comm_export.py +++ b/pyzebra/comm_export.py @@ -30,42 +30,42 @@ def export_comm(data, path, lorentz=False): padding = [4, 6, 10, 8] with open(str(path + extension), "w") as out_file: - for key, meas in data["meas"].items(): - if "fit" not in meas: - print("Measurement skipped - no fit value for:", key) + for key, scan in data["scan"].items(): + if "fit" not in scan: + print("Scan skipped - no fit value for:", key) continue - meas_number_str = f"{key:{align}{padding[0]}}" - h_str = f'{int(meas["h_index"]):{padding[1]}}' - k_str = f'{int(meas["k_index"]):{padding[1]}}' - l_str = f'{int(meas["l_index"]):{padding[1]}}' + scan_number_str = f"{key:{align}{padding[0]}}" + h_str = f'{int(scan["h_index"]):{padding[1]}}' + k_str = f'{int(scan["k_index"]):{padding[1]}}' + l_str = f'{int(scan["l_index"]):{padding[1]}}' if data["meta"]["area_method"] == "fit": - area = float(meas["fit"]["fit_area"].n) + area = float(scan["fit"]["fit_area"].n) sigma_str = ( - f'{"{:8.2f}".format(float(meas["fit"]["fit_area"].s)):{align}{padding[2]}}' + f'{"{:8.2f}".format(float(scan["fit"]["fit_area"].s)):{align}{padding[2]}}' ) elif data["meta"]["area_method"] == "integ": - area = float(meas["fit"]["int_area"].n) + area = float(scan["fit"]["int_area"].n) sigma_str = ( - f'{"{:8.2f}".format(float(meas["fit"]["int_area"].s)):{align}{padding[2]}}' + f'{"{:8.2f}".format(float(scan["fit"]["int_area"].s)):{align}{padding[2]}}' ) if zebra_mode == "bi": - area = correction(area, lorentz, zebra_mode, meas["twotheta_angle"]) + area = correction(area, lorentz, zebra_mode, scan["twotheta_angle"]) int_str = f'{"{:8.2f}".format(area):{align}{padding[2]}}' - angle_str1 = f'{meas["twotheta_angle"]:{padding[3]}}' - angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' - angle_str3 = f'{meas["chi_angle"]:{padding[3]}}' - angle_str4 = f'{meas["phi_angle"]:{padding[3]}}' + angle_str1 = f'{scan["twotheta_angle"]:{padding[3]}}' + angle_str2 = f'{scan["omega_angle"]:{padding[3]}}' + angle_str3 = f'{scan["chi_angle"]:{padding[3]}}' + angle_str4 = f'{scan["phi_angle"]:{padding[3]}}' elif zebra_mode == "nb": - area = correction(area, lorentz, zebra_mode, meas["gamma_angle"], meas["nu_angle"]) + area = correction(area, lorentz, zebra_mode, scan["gamma_angle"], scan["nu_angle"]) int_str = f'{"{:8.2f}".format(area):{align}{padding[2]}}' - angle_str1 = f'{meas["gamma_angle"]:{padding[3]}}' - angle_str2 = f'{meas["omega_angle"]:{padding[3]}}' - angle_str3 = f'{meas["nu_angle"]:{padding[3]}}' - angle_str4 = f'{meas["unkwn_angle"]:{padding[3]}}' + angle_str1 = f'{scan["gamma_angle"]:{padding[3]}}' + angle_str2 = f'{scan["omega_angle"]:{padding[3]}}' + angle_str3 = f'{scan["nu_angle"]:{padding[3]}}' + angle_str4 = f'{scan["unkwn_angle"]:{padding[3]}}' line = ( - meas_number_str + scan_number_str + h_str + l_str + k_str diff --git a/pyzebra/fit2.py b/pyzebra/fit2.py index 242149a..c4132e6 100644 --- a/pyzebra/fit2.py +++ b/pyzebra/fit2.py @@ -32,7 +32,7 @@ def create_uncertanities(y, y_err): def fitccl( - meas, + scan, guess, vary, constraints_min, @@ -42,7 +42,7 @@ def fitccl( binning=None, ): """Made for fitting of ccl date where 1 peak is expected. Allows for combination of gaussian and linear model combination - :param meas: measurement in the data dict (i.e. M123) + :param scan: scan in the data dict (i.e. M123) :param guess: initial guess for the fitting, if none, some values are added automatically in order (see below) :param vary: True if parameter can vary during fitting, False if it to be fixed :param numfit_min: minimal value on x axis for numerical integration - if none is centre of gaussian minus 3 sigma @@ -59,33 +59,33 @@ def fitccl( constraints_min = [23, None, 50, 0, 0] constraints_min = [80, None, 1000, 0, 100] """ - if len(meas["peak_indexes"]) > 1: + if len(scan["peak_indexes"]) > 1: # return in case of more than 1 peaks - print("More than 1 peak, measurement skipped") + print("More than 1 peak, scan skipped") return if binning is None or binning == 0 or binning == 1: - x = list(meas["om"]) - y = list(meas["Counts"]) - y_err = list(np.sqrt(y)) if meas.get("sigma", None) is None else list(meas["sigma"]) - print(meas["peak_indexes"]) - if not meas["peak_indexes"]: + x = list(scan["om"]) + y = list(scan["Counts"]) + y_err = list(np.sqrt(y)) if scan.get("sigma", None) is None else list(scan["sigma"]) + print(scan["peak_indexes"]) + if not scan["peak_indexes"]: centre = np.mean(x) else: - centre = x[int(meas["peak_indexes"])] + centre = x[int(scan["peak_indexes"])] else: - x = list(meas["om"]) - if not meas["peak_indexes"]: + x = list(scan["om"]) + if not scan["peak_indexes"]: centre = np.mean(x) else: - centre = x[int(meas["peak_indexes"])] + centre = x[int(scan["peak_indexes"])] x = bin_data(x, binning) - y = list(meas["Counts"]) - y_err = list(np.sqrt(y)) if meas.get("sigma", None) is None else list(meas["sigma"]) + y = list(scan["Counts"]) + y_err = list(np.sqrt(y)) if scan.get("sigma", None) is None else list(scan["sigma"]) combined = bin_data(create_uncertanities(y, y_err), binning) y = [combined[i].n for i in range(len(combined))] y_err = [combined[i].s for i in range(len(combined))] - if len(meas["peak_indexes"]) == 0: + if len(scan["peak_indexes"]) == 0: # Case for no peak, gaussian in centre, sigma as 20% of range print("No peak") peak_index = find_nearest(x, np.mean(x)) @@ -96,10 +96,10 @@ def fitccl( guess[4] = np.mean(y) if guess[4] is None else guess[4] constraints_min[2] = 0 - elif len(meas["peak_indexes"]) == 1: + elif len(scan["peak_indexes"]) == 1: # case for one peak, takse into account users guesses print("one peak") - peak_height = meas["peak_heights"] + peak_height = scan["peak_heights"] guess[0] = centre if guess[0] is None else guess[0] guess[1] = 0.1 if guess[1] is None else guess[1] guess[2] = float(peak_height / 10) if guess[2] is None else float(guess[2]) @@ -144,11 +144,11 @@ def fitccl( fit_area = u.ufloat(result.params["g_amp"].value, result.params["g_amp"].stderr) comps = result.eval_components() - if len(meas["peak_indexes"]) == 0: + if len(scan["peak_indexes"]) == 0: # for case of no peak, there is no reason to integrate, therefore fit and int are equal int_area = fit_area - elif len(meas["peak_indexes"]) == 1: + elif len(scan["peak_indexes"]) == 1: gauss_3sigmamin = find_nearest( x, result.params["g_cen"].value - 3 * result.params["g_width"].value ) @@ -224,4 +224,4 @@ def fitccl( d["result"] = result d["comps"] = comps d["numfit"] = [numfit_min, numfit_max] - meas["fit"] = d + scan["fit"] = d diff --git a/pyzebra/load_1D.py b/pyzebra/load_1D.py index f208a89..50f3603 100644 --- a/pyzebra/load_1D.py +++ b/pyzebra/load_1D.py @@ -58,7 +58,7 @@ META_VARS_FLOAT = ( META_UB_MATRIX = ("ub1j", "ub2j", "ub3j") CCL_FIRST_LINE = ( - # the first element is `measurement_number`, which we don't save to metadata + # the first element is `scan_number`, which we don't save to metadata ("h_index", float), ("k_index", float), ("l_index", float), @@ -100,8 +100,8 @@ def load_1D(filepath): :arg filepath :returns det_variables - - dictionary of all detector/scan variables and dictinionary for every measurement. - Names of these dictionaries are M + measurement number. They include HKL indeces, angles, + - dictionary of all detector/scan variables and dictinionary for every scan. + Names of these dictionaries are M + scan number. They include HKL indeces, angles, monitors, stepsize and array of counts """ with open(filepath, "r") as infile: @@ -130,7 +130,7 @@ def parse_1D(fileobj, data_type): break # read data - measurements = {} + scan = {} if data_type == ".ccl": decimal = list() @@ -144,7 +144,7 @@ def parse_1D(fileobj, data_type): d = {} # first line - measurement_number, *params = line.split() + scan_number, *params = line.split() for param, (param_name, param_type) in zip(params, ccl_first_line): d[param_name] = param_type(param) @@ -170,7 +170,7 @@ def parse_1D(fileobj, data_type): counts.extend(map(int, next(fileobj).split())) d["Counts"] = counts - measurements[int(measurement_number)] = d + scan[int(scan_number)] = d if all(decimal): metadata["indices"] = "hkl" @@ -209,7 +209,7 @@ def parse_1D(fileobj, data_type): data_cols["phi_angle"] = metadata["phi"] data_cols["nu_angle"] = metadata["nu"] - measurements[1] = dict(data_cols) + scan[1] = dict(data_cols) else: print("Unknown file extention") @@ -218,4 +218,4 @@ def parse_1D(fileobj, data_type): metadata["data_type"] = data_type metadata["area_method"] = "fit" - return {"meta": metadata, "meas": measurements} + return {"meta": metadata, "scan": scan} diff --git a/pyzebra/param_study_moduls.py b/pyzebra/param_study_moduls.py index c249139..2593039 100644 --- a/pyzebra/param_study_moduls.py +++ b/pyzebra/param_study_moduls.py @@ -38,12 +38,10 @@ def load_dats(filepath): dict1 = add_dict(dict1, load_1D(file_list[i][0])) else: dict1 = add_dict(dict1, load_1D(file_list[i])) - dict1["meas"][i + 1]["params"] = {} + dict1["scan"][i + 1]["params"] = {} if data_type == "txt": for x in range(len(col_names) - 1): - dict1["meas"][i + 1]["params"][ - col_names[x + 1] - ] = file_list[i][x + 1] + dict1["scan"][i + 1]["params"][col_names[x + 1]] = file_list[i][x + 1] return dict1 @@ -55,7 +53,7 @@ def create_dataframe(dict1): # create dictionary to which we pull only wanted items before transforming it to pd.dataframe pull_dict = {} pull_dict["filenames"] = list() - for key in dict1["meas"][1]["params"]: + for key in dict1["scan"][1]["params"]: pull_dict[key] = list() pull_dict["temperature"] = list() pull_dict["mag_field"] = list() @@ -65,21 +63,19 @@ def create_dataframe(dict1): pull_dict["Counts"] = list() # populate the dict - for keys in dict1["meas"]: - if "file_of_origin" in dict1["meas"][keys]: - pull_dict["filenames"].append( - dict1["meas"][keys]["file_of_origin"].split("/")[-1] - ) + for keys in dict1["scan"]: + if "file_of_origin" in dict1["scan"][keys]: + pull_dict["filenames"].append(dict1["scan"][keys]["file_of_origin"].split("/")[-1]) else: pull_dict["filenames"].append(dict1["meta"]["original_filename"].split("/")[-1]) - for key in dict1["meas"][keys]["params"]: - pull_dict[str(key)].append(float(dict1["meas"][keys]["params"][key])) - pull_dict["temperature"].append(dict1["meas"][keys]["temperature"]) - pull_dict["mag_field"].append(dict1["meas"][keys]["mag_field"]) - pull_dict["fit_area"].append(dict1["meas"][keys]["fit"]["fit_area"]) - pull_dict["int_area"].append(dict1["meas"][keys]["fit"]["int_area"]) - pull_dict["om"].append(dict1["meas"][keys]["om"]) - pull_dict["Counts"].append(dict1["meas"][keys]["Counts"]) + for key in dict1["scan"][keys]["params"]: + pull_dict[str(key)].append(float(dict1["scan"][keys]["params"][key])) + pull_dict["temperature"].append(dict1["scan"][keys]["temperature"]) + pull_dict["mag_field"].append(dict1["scan"][keys]["mag_field"]) + pull_dict["fit_area"].append(dict1["scan"][keys]["fit"]["fit_area"]) + pull_dict["int_area"].append(dict1["scan"][keys]["fit"]["int_area"]) + pull_dict["om"].append(dict1["scan"][keys]["om"]) + pull_dict["Counts"].append(dict1["scan"][keys]["Counts"]) return pd.DataFrame(data=pull_dict) @@ -145,12 +141,14 @@ def make_graph(data, sorting_parameter, style): plt.clim(color_matrix.mean(), color_matrix.max()) return fig + + def save_dict(obj, name): """ saves dictionary as pickle file in binary format :arg obj - object to save :arg name - name of the file NOTE: path should be added later""" - with open(name + '.pkl', 'wb') as f: + with open(name + ".pkl", "wb") as f: pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL) @@ -159,15 +157,17 @@ def load_dict(name): :arg name - name of the file to load NOTE: expect the file in the same folder, path should be added later :return dictionary""" - with open(name + '.pkl', 'rb') as f: + with open(name + ".pkl", "rb") as f: return pickle.load(f) + + # pickle, mat, h5, txt, csv, json def save_table(data, filetype, name, path=None): print("Saving: ", filetype) path = "" if path is None else path if filetype == "pickle": # to work with uncertanities, see uncertanity module - with open(path + name + '.pkl', 'wb') as f: + with open(path + name + ".pkl", "wb") as f: pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) if filetype == "mat": # matlab doesent allow some special character to be in var names, also cant start with @@ -176,21 +176,23 @@ def save_table(data, filetype, name, path=None): data["fit_area_err"] = [data["fit_area"][i].s for i in range(len(data["fit_area"]))] data["int_area_nom"] = [data["int_area"][i].n for i in range(len(data["int_area"]))] data["int_area_err"] = [data["int_area"][i].s for i in range(len(data["int_area"]))] - data = data.drop(columns=['fit_area', 'int_area']) - remove_characters = [" ", "[", "]", "{", "}", "(",")"] + data = data.drop(columns=["fit_area", "int_area"]) + remove_characters = [" ", "[", "]", "{", "}", "(", ")"] for character in remove_characters: - data.columns = [data.columns[i].replace(character,"") for i in range(len(data.columns))] - sio.savemat((path + name + '.mat'), {name: col.values for name, col in data.items()}) + data.columns = [ + data.columns[i].replace(character, "") for i in range(len(data.columns)) + ] + sio.savemat((path + name + ".mat"), {name: col.values for name, col in data.items()}) if filetype == "csv" or "txt": data["fit_area_nom"] = [data["fit_area"][i].n for i in range(len(data["fit_area"]))] data["fit_area_err"] = [data["fit_area"][i].s for i in range(len(data["fit_area"]))] data["int_area_nom"] = [data["int_area"][i].n for i in range(len(data["int_area"]))] data["int_area_err"] = [data["int_area"][i].s for i in range(len(data["int_area"]))] - data = data.drop(columns=['fit_area', 'int_area', 'om', 'Counts']) + data = data.drop(columns=["fit_area", "int_area", "om", "Counts"]) if filetype == "csv": - data.to_csv(path + name + '.csv') + data.to_csv(path + name + ".csv") if filetype == "txt": - with open((path + name + '.txt'), 'w') as outfile: + with open((path + name + ".txt"), "w") as outfile: data.to_string(outfile) if filetype == "h5": hdf = pd.HDFStore((path + name + ".h5")) @@ -198,9 +200,3 @@ def save_table(data, filetype, name, path=None): hdf.close() if filetype == "json": data.to_json((path + name + ".json")) - - - - - - From 7bc0ec04387d31532717fdbebd3f769def743362 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Tue, 13 Oct 2020 14:09:27 +0200 Subject: [PATCH 115/116] Display omega in hovertool --- pyzebra/app/panel_hdf_viewer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyzebra/app/panel_hdf_viewer.py b/pyzebra/app/panel_hdf_viewer.py index c0e1409..956506a 100644 --- a/pyzebra/app/panel_hdf_viewer.py +++ b/pyzebra/app/panel_hdf_viewer.py @@ -87,7 +87,8 @@ def create(): temperature_spinner.value = det_data["temperature"][index] gamma, nu = calculate_pol(det_data, index) - image_source.data.update(gamma=[gamma], nu=[nu]) + omega = det_data["rot_angle"][index] + image_source.data.update(gamma=[gamma], nu=[nu], omega=[omega]) def update_overview_plot(): h5_data = det_data["data"] @@ -161,6 +162,7 @@ def create(): l=[np.zeros((1, 1))], gamma=[np.zeros((1, 1))], nu=[np.zeros((1, 1))], + omega=[np.zeros((1, 1))], x=[0], y=[0], dw=[IMAGE_W], @@ -173,12 +175,14 @@ def create(): l_glyph = Image(image="l", x="x", y="y", dw="dw", dh="dh", global_alpha=0) gamma_glyph = Image(image="gamma", x="x", y="y", dw="dw", dh="dh", global_alpha=0) nu_glyph = Image(image="nu", x="x", y="y", dw="dw", dh="dh", global_alpha=0) + omega_glyph = Image(image="omega", x="x", y="y", dw="dw", dh="dh", global_alpha=0) plot.add_glyph(image_source, h_glyph) plot.add_glyph(image_source, k_glyph) plot.add_glyph(image_source, l_glyph) plot.add_glyph(image_source, gamma_glyph) plot.add_glyph(image_source, nu_glyph) + plot.add_glyph(image_source, omega_glyph) image_glyph = Image(image="image", x="x", y="y", dw="dw", dh="dh") plot.add_glyph(image_source, image_glyph, name="image_glyph") @@ -224,6 +228,7 @@ def create(): ("intensity", "@image"), ("gamma", "@gamma"), ("nu", "@nu"), + ("omega", "@omega"), ("h", "@h"), ("k", "@k"), ("l", "@l"), From ee788ab48a9fcbe186f79138155ba0255c6e4582 Mon Sep 17 00:00:00 2001 From: Ivan Usov Date: Wed, 21 Oct 2020 14:04:53 +0200 Subject: [PATCH 116/116] Fix image not updating --- pyzebra/app/panel_hdf_viewer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyzebra/app/panel_hdf_viewer.py b/pyzebra/app/panel_hdf_viewer.py index 956506a..2ff09d9 100644 --- a/pyzebra/app/panel_hdf_viewer.py +++ b/pyzebra/app/panel_hdf_viewer.py @@ -87,7 +87,7 @@ def create(): temperature_spinner.value = det_data["temperature"][index] gamma, nu = calculate_pol(det_data, index) - omega = det_data["rot_angle"][index] + omega = np.ones((IMAGE_H, IMAGE_W)) * det_data["rot_angle"][index] image_source.data.update(gamma=[gamma], nu=[nu], omega=[omega]) def update_overview_plot():