doc/requirements.txt

+sphinx
+sphinx_rtd_theme
+autoapi
+sphinx-autoapi
+nbsphinx
+urllib3<2.0.0

doc/scripts/bin_dssc_module_job.sh

+#!/bin/bash
+#SBATCH -N 1
+#SBATCH --partition=exfel
+#SBATCH --time=12:00:00
+#SBATCH --mail-type=END,FAIL
+#SBATCH --output=logs/%j-%x.out
+
+while getopts ":p:d:r:k:m:x:b:" option
+do
+    case $option in
+        p) PROPOSAL="$OPTARG";;
+        d) DARK="$OPTARG";;
+        r) RUN="$OPTARG";;
+        k) KERNEL="$OPTARG";;
+        m) MODULE_GROUP="$OPTARG";;
+        x) XAXIS="$OPTARG";;
+        b) BINWIDTH="$OPTARG";;
+        \?) echo "Unknown option"
+            exit 1;;
+        :) echo "Missing option for input flag"
+           exit 1;;
+    esac
+done
+
+# Load xfel environment
+source /etc/profile.d/modules.sh
+module load exfel exfel-python
+
+echo processing run $RUN
+PDIR=$(findxfel $PROPOSAL)
+PPROPOSAL="p$(printf '%06d' $PROPOSAL)"
+RDIR="$PDIR/usr/processed_runs/r$(printf '%04d' $RUN)"
+mkdir $RDIR
+
+NB='Dask DSSC module binning.ipynb'
+
+# kernel list can be seen from 'jupyter kernelspec list'
+if [ -z "${KERNEL}" ]; then
+	  KERNEL="toolbox_$PPROPOSAL"
+fi
+
+python -c "import papermill as pm; pm.execute_notebook(\
+  '$NB', \
+  '$RDIR/output$MODULE_GROUP.ipynb', \
+  kernel_name='$KERNEL', \
+  parameters=dict(proposalNB=int('$PROPOSAL'), \
+                  dark_runNB=int('$DARK'), \
+                  runNB=int('$RUN'), \
+                  module_group=int('$MODULE_GROUP'), \
+                  path='$RDIR/', \
+                  xaxis='$XAXIS', \
+                  bin_width=float('$BINWIDTH')))"

doc/scripts/boz_parameters_job.sh

+#!/bin/bash
+#SBATCH -N 1
+#SBATCH --partition=allgpu
+#SBATCH --constraint=V100
+#SBATCH --time=2:00:00
+#SBATCH --mail-type=END,FAIL
+#SBATCH --output=logs/%j-%x.out
+
+ROISTH='1'
+SATLEVEL='500'
+MODULE='15'
+
+while getopts ":p:d:r:k:g:t:s:m:" option
+do
+    case $option in
+        p) PROPOSAL="$OPTARG";;
+        d) DARK="$OPTARG";;
+        r) RUN="$OPTARG";;
+        k) KERNEL="$OPTARG";;
+        g) GAIN="$OPTARG";;
+        t) ROISTH="$OPTARG";;
+        s) SATLEVEL="$OPTARG";;
+        m) MODULE="$OPTARG";;
+        \?) echo "Unknown option"
+            exit 1;;
+        :) echo "Missing option for input flag"
+           exit 1;;
+    esac
+done
+
+# Load xfel environment
+source /etc/profile.d/modules.sh
+module load exfel exfel-python
+
+echo processing run $RUN
+PDIR=$(findxfel $PROPOSAL)
+PPROPOSAL="p$(printf '%06d' $PROPOSAL)"
+RDIR="$PDIR/usr/processed_runs/r$(printf '%04d' $RUN)"
+mkdir $RDIR
+
+NB='BOZ analysis part I.a Correction determination.ipynb'
+
+# kernel list can be seen from 'jupyter kernelspec list'
+if [ -z "${KERNEL}" ]; then
+	KERNEL="toolbox_$PPROPOSAL"
+fi
+
+python -c "import papermill as pm; pm.execute_notebook(\
+  '$NB', \
+  '$RDIR/output.ipynb', \
+  kernel_name='$KERNEL', \
+  parameters=dict(proposal=int('$PROPOSAL'), \
+                  darkrun=int('$DARK'), \
+                  run=int('$RUN'), \
+                  module=int('$MODULE'), \
+                  gain=float('$GAIN'), \
+                  rois_th=float('$ROISTH'), \
+                  sat_level=int('$SATLEVEL')))"

doc/scripts/format_data.py

+import os
+import logging
+import argparse
+
+import numpy as np
+
+import toolbox_scs as tb
+import toolbox_scs.detectors as tbdet
+
+logging.basicConfig(level=logging.INFO)
+log_root = logging.getLogger(__name__)
+
+# -----------------------------------------------------------------------------
+# user input:
+# -----------------------------------------------------------------------------
+run_type = 'static, delay, .....'
+description = 'useful description or comment .....'
+#add xgm data to formatted file if save_xgm_binned was set to True
+metadata = ['binner1', 'binner2', 'xgm_binned'] # ['binner1', 'binner2'] 
+# -----------------------------------------------------------------------------
+
+
+def formatting(run_number, run_folder):
+    log_root.debug("Collect, combine and format files in run folder")
+
+    run_formatted = tbdet.DSSCFormatter(run_folder)
+    run_formatted.combine_files()
+
+    run_formatted.add_dataArray(metadata)
+
+    attrs = {'run_type':run_type,
+             'description':description,
+             'run_number':run_number}
+    run_formatted.add_attributes(attrs)
+
+    run_formatted.save_formatted_data(f'{run_folder}run_{run_number}_formatted.h5')
+    log_root.debug("Formatting finished successfully.")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--run-number', metavar='S',
+                        action='store',
+                        help='run number')
+    parser.add_argument('--run-folder', metavar='S',
+                        action='store',
+                        help='the run folder containing fractional data')
+    args = parser.parse_args()
+
+    formatting(str(args.run_number), str(args.run_folder))

doc/scripts/format_data.sh

+#!/bin/bash
+
+RUN_NR=${1}
+RUN_DIR="../processed_runs/r_${RUN_NR}/"
+
+
+if [ -d $RUN_DIR ]
+    then 
+        echo creating formatted .h5 file for run $RUN_NR in $RUN_DIR
+	source /etc/profile.d/modules.sh
+        module load exfel exfel-python
+
+        python format_data.py --run-number $RUN_NR --run-folder $RUN_DIR
+        #chgrp -R 60002711-part $RUN_DIR
+        chmod -R 777 $RUN_DIR
+    else
+        echo run folder $RUN_DIR does not exist
+        echo please provide a valid run number
+fi

doc/scripts/process_data_201007_23h.py

+import os
+import logging
+import argparse
+import h5py
+
+import numpy as np
+import extra_data as ed
+
+import toolbox_scs as tb
+import toolbox_scs.detectors as tbdet
+
+logging.basicConfig(level=logging.INFO)
+log_root = logging.getLogger(__name__)
+
+
+# -----------------------------------------------------------------------------
+# user input: run-type specific
+# -----------------------------------------------------------------------------
+proposal_nb = 2599
+output_filepath = "../processed_runs/"
+
+# these get set by the shell script now! (e.g. "--runtype static")
+# runtype = 'energyscan'
+# runtype = 'energyscan_pumped'
+# runtype = 'static'
+# runtype = 'static_IR'
+# runtype = 'delayscan'
+# runtype = 'timescan'
+
+# useful metadata to be added to h5 files
+scriptname = os.path.basename(__file__)
+
+save_xgm_binned = True
+
+# optional prebinning methods for DSSC data
+normevery = 2 # 2 if use intradark, 1 otherwise
+xgm_mask = True  # True: xgm_threshold will be used to drop corresponding DSSC frames accordingly to the xgm treshold
+xgm_threshold = (1000, np.inf) # or you mean bad pulses here ?
+filename_dark = None # 200
+xgm_normalization = False 
+
+# -----------------------------------------------------------------------------
+
+
+def process(run_nb, runtype, modules=[]):
+    run_description = f'{runtype}; script {scriptname}'
+    print(run_description)
+    mod_list = modules
+    if len(mod_list)==0:
+        mod_list = [i for i in range(16)]
+
+    path = f'{output_filepath}r_{run_nb}/'
+    log_root.info("create run objects")
+    run_info = tbdet.load_dssc_info(proposal_nb, run_nb)
+    fpt = run_info['frames_per_train']
+    n_trains = run_info['number_of_trains']
+    trainIds = run_info['trainIds']
+
+    # -------------------------------------------------------------------------
+    # user input: run specific
+    # -------------------------------------------------------------------------
+    run_obj = ed.open_run(proposal_nb, run_nb)
+    
+    if runtype == 'static':
+        buckets_train = np.zeros(n_trains)
+        pulsepattern = ['image', 'intradark']
+        buckets_pulse = pulsepattern * (fpt // len(pulsepattern))
+    
+    if runtype == 'energyscan':
+        buckets_train = tb.get_array(run_obj, 'nrj', 0.1).values
+        pulsepattern = ['image', 'intradark']
+        buckets_pulse = pulsepattern * (fpt // len(pulsepattern))
+    
+    if runtype == 'static_IR':
+        buckets_train = np.zeros(n_trains)
+        pulsepattern = ['unpumped', 'unpumped_intradark', 'pumped', 'pumped_intradark']
+        buckets_pulse = pulsepattern * (fpt // len(pulsepattern))
+        
+    if runtype == 'energyscan_pumped':
+        buckets_train = tb.get_array(run_obj, 'nrj', 0.1).values
+        pulsepattern = ['unpumped', 'unpumped_intradark', 'pumped', 'pumped_intradark']
+        buckets_pulse = pulsepattern * (fpt // len(pulsepattern))
+        
+    if runtype == 'delayscan':
+        buckets_train = tb.get_array(run_obj, 'PP800_DelayLine', 0.03).values
+        pulsepattern = ['unpumped', 'unpumped_intradark', 'pumped', 'pumped_intradark']
+        buckets_pulse = pulsepattern * (fpt // len(pulsepattern))
+    
+    if runtype == 'timescan':  # 10s bins (tstamp is in ns)
+        bin_nsec = 10 * 1e9
+        tstamp = run_obj.get_array('SCS_RR_UTC/TSYS/TIMESERVER', 'id.timestamp')
+        buckets_train = (bin_nsec * np.round(tstamp / bin_nsec) - tstamp.min()) / 1e9
+        pulsepattern = ['unpumped', 'unpumped_intradark', 'pumped', 'pumped_intradark']
+        buckets_pulse = pulsepattern * (fpt // len(pulsepattern))
+    # -------------------------------------------------------------------------
+
+    # create binner
+    binner1 = tbdet.create_dssc_bins("trainId",trainIds,buckets_train)
+    binner2 = tbdet.create_dssc_bins("pulse",
+                                     np.linspace(0,fpt-1,fpt, dtype=int),
+                                     buckets_pulse)
+    binners = {'trainId': binner1, 'pulse': binner2}
+    bin_obj = tbdet.DSSCBinner(proposal_nb, run_nb,
+                               binners=binners,
+                               dssc_coords_stride=normevery)
+    
+    if xgm_mask:
+        bin_obj.create_pulsemask('xgm', xgm_threshold)
+
+    dark=None
+    if filename_dark:
+        dark = tbdet.load_xarray(filename_dark)
+        dark = dark['data']
+
+    bin_params = {'modules':mod_list,
+                  'chunksize':248,
+                  'filepath':path,
+                  'xgm_normalization':xgm_normalization,
+                  'normevery':normevery,
+                  'dark_image':dark}
+
+    log_root.info("start binning routine")
+    bin_obj.process_data(**bin_params)
+
+    log_root.info("Add additional data to module files")
+    if save_xgm_binned:
+        bin_obj.load_xgm()
+        xgm_binned = bin_obj.get_xgm_binned()
+
+    if not os.path.isdir(path):
+        os.mkdir(path)
+    for m in mod_list:
+        fname = f'run_{run_nb}_module{m}.h5'
+        if save_xgm_binned:
+            tbdet.save_xarray(
+                path+fname, xgm_binned, group='xgm_binned', mode='a')
+        tbdet.save_xarray(path+fname, binner1, group='binner1', mode='a')
+        tbdet.save_xarray(path+fname, binner2, group='binner2', mode='a')
+        metadata = {'run_number':run_nb,
+                    'module':m,
+                    'run_description':run_description}
+        tbdet.save_attributes_h5(path+fname, metadata)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--run-number', metavar='S',
+                        action='store',
+                        help='the run to be processed')
+    parser.add_argument('--module', metavar='S',
+                        nargs='+', action='store',
+                        help='modules to be processed')
+    parser.add_argument('--runtype', metavar='S',
+                        nargs='+', action='store',
+                        help=('type of run (static, static_IR, energyscan, energyscan_pumped)'
+                              ', delayscan', 'timescan)'))
+    args = parser.parse_args()
+    
+    runtype = args.runtype[0]
+    if args.run_number:
+        if args.module is not None:
+            modules = []
+            if len(args.module) == 1:
+                args.module = args.module[0].split(" ")
+            modules = list(map(int, args.module))
+            process(str(args.run_number), runtype, modules)
+        else:
+            process(str(args.run_number), runtype)

doc/scripts/start_job_single.sh

+#!/bin/bash
+#SBATCH -N 1
+#SBATCH --partition=upex
+#SBATCH --time=00:30:00
+#SBATCH --mail-type=END,FAIL
+#SBATCH --output=../logs/%j-%x.out
+
+RUN=$1
+MODULES=$2
+RUNTYPE=$3
+
+source /etc/profile.d/modules.sh
+module load exfel exfel-python
+
+echo processing modules $MODULES of run $RUN
+python process_data_201007_23h.py --run-number $RUN --module ${MODULES} --runtype $RUNTYPE

doc/scripts/start_processing_all.sh

+#!/bin/bash
+
+RUN=$1
+RUNTYPE=$2
+
+if [ $RUN ] && [ $RUNTYPE ]
+    then
+        echo processing run $RUN
+        source /etc/profile.d/modules.sh
+        module load exfel exfel-python
+        
+        sbatch ./start_job_single.sh $RUN '0 1 2 3' $RUNTYPE
+        sbatch ./start_job_single.sh $RUN '4 5 6 7' $RUNTYPE
+        sbatch ./start_job_single.sh $RUN '8 9 10 11' $RUNTYPE
+        sbatch ./start_job_single.sh $RUN '12 13 14 15' $RUNTYPE
+    else
+        echo please specify a run number and type
+        echo available runtypes:
+        echo energyscan, energyscan_pumped, static, static_IR, delayscan, timescan
+fi

doc/transient reflectivity.rst

+Finding time overlap by transient reflectivity
+----------------------------------------------
+
+Transient reflectivity of the optical laser measured on a large bandgap material pumped by the FEL is often used at SCS to find the time overlap between the two beams. The example notebook 
+
+* :doc:`Transient reflectivity measurement <Transient reflectivity measurement>`
+
+shows how to analyze such data, including correcting the delay by the bunch arrival monitor (BAM).
+

notebook_examples/XAS and XMCD energy shift investigation.ipynb

+%% Cell type:markdown id: tags:
+
+This notebook shows an example of X-ray absorption spectroscopy (XAS) and X-ray magnetic circular dichroism (XMCD) experiment.
+
+%% Cell type:markdown id: tags:
+
+# Import toolbox_scs and subpackages
+
+%% Cell type:code id: tags:
+
+``` python
+import numpy as np
+%matplotlib notebook
+import matplotlib.pyplot as plt
+plt.rcParams['figure.constrained_layout.use'] = True
+
+import toolbox_scs as tb
+import toolbox_scs.detectors as tbdet
+import toolbox_scs.routines as tbr
+import extra_data as ed
+import xarray as xr
+
+import logging
+logging.basicConfig(level=logging.INFO)
+log_root = logging.getLogger(__name__)
+```
+
+%% Cell type:markdown id: tags:
+
+# Load data, extract pulse-resolved signals from detectors
+
+Here, we use the ToolBox mnemonics to get the arrays: MCP1apd is the pulse-resolved peak-integrated TIM data from MCP 1, SCS_SA3 is the XGM data for SASE 3 pulses in SCS hutch, nrj is the monochromator energy in eV and magnet is the current applied to the magnet in A.
+
+The function get_all_detectors() takes care of aligning the pulse-resolved data (TIM and XGM) according to the sase 3 bunch pattern.
+
+%% Cell type:code id: tags:
+
+``` python
+runNB = 491
+proposalNB = 900094
+mcp = 'MCP1apd'
+fields = [mcp, 'SCS_SA3', 'nrj', 'magnet']
+run, ds = tb.load(proposalNB, runNB, fields)
+mcp = mcp.replace('apd', 'peaks')
+ds
+```
+
+%% Cell type:markdown id: tags:
+
+## Split positive and negative magnetic fields, calculate XAS and XMCD spectra
+
+Here we first define the monochromator energy bins, and apply the xas() subroutine to subsets of the data. The subsets correspond to positive magnet current and negative magnet current. The xas() routine returns (among other quantities) the binned energy and the absorption, which are then plotted for both positive and negative magnet currents.
+
+%% Cell type:code id: tags:
+
+``` python
+nrj_bins_edges = np.linspace(704, 712, 31)
+pxas = tbr.xas(ds.where(ds['magnet'] > 2, drop=True), bins=nrj_bins_edges, plot=False, Itkey=mcp)
+nxas = tbr.xas(ds.where(ds['magnet'] < -2, drop=True), bins=nrj_bins_edges, plot=False, Itkey=mcp)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+plt.figure()
+plt.plot(pxas['nrj'], pxas['muA'])
+plt.plot(nxas['nrj'], nxas['muA'])
+plt.ylabel('$\mu_A$')
+plt.xlabel('Energy [eV]')
+```
+
+%% Cell type:markdown id: tags:
+
+We now calculate the XMCD using the positive and negative data returned by xas() routine.
+
+%% Cell type:code id: tags:
+
+``` python
+res = tbr.xasxmcd(pxas, nxas)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+plt.figure()
+plt.plot(res['nrj'], res['muXAS'])
+plt.ylabel('$\mu_A$')
+plt.xlabel('Energy [eV]')
+plt.twinx()
+plt.plot(res['nrj'], -res['muXMCD'], c='C1')
+plt.ylabel('$\mu_{XMCD}$')
+```
+
+%% Cell type:markdown id: tags:
+
+# Only 2000 first train
+
+%% Cell type:code id: tags:
+
+``` python
+subds = ds.isel({'trainId':slice(1600)})
+
+pxas2 = tbr.xas(subds.where(subds['magnet'] > 2, drop=True), bins=nrj_bins_edges, plot=False, Itkey=mcp)
+nxas2 = tbr.xas(subds.where(subds['magnet'] < -2, drop=True), bins=nrj_bins_edges, plot=False, Itkey=mcp)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+plt.figure()
+plt.plot(subds['nrj'])
+plt.ylabel('energy [eV]')
+plt.xlabel('train number')
+plt.twinx()
+plt.plot(subds['magnet'], c='C1')
+plt.ylabel('magnet current [A]')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+plt.figure()
+plt.errorbar(pxas['nrj'], pxas['muA'], pxas['sterrA'], label='pos all trains')
+plt.errorbar(pxas2['nrj'], pxas2['muA'], pxas2['sterrA'], c='C0', ls='--', label='pos first 2000 trains')
+
+plt.errorbar(nxas['nrj'], nxas['muA'], nxas['sterrA'], label='neg all trains')
+plt.errorbar(nxas2['nrj'], nxas2['muA'], nxas['sterrA'], c='C1', ls='--', label='neg first 2000 trains')
+plt.legend()
+plt.ylabel('$\mu_A$')
+plt.xlabel('Energy [eV]')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+res2 = tbr.xasxmcd(pxas2, nxas2)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+plt.figure()
+plt.plot(res['nrj'], res['muXAS'])
+plt.plot(res2['nrj'], res2['muXAS'], ls='--', c='C0')
+plt.ylabel('$\mu_A$')
+plt.xlabel('Energy [eV]')
+plt.twinx()
+plt.plot(res['nrj'], -res['muXMCD'], c='C1')
+plt.plot(res2['nrj'], -res2['muXMCD'], ls='--', c='C1')
+plt.ylabel('$\mu_{XMCD}$')
+```
+
+%% Cell type:markdown id: tags:
+
+# Mono up or down
+
+%% Cell type:code id: tags:
+
+``` python
+from scipy.signal import savgol_filter
+
+dE = np.gradient(savgol_filter(ds['nrj'], 9, 1))
+mono_turn = dE > 0
+
+plt.figure()
+plt.plot(1e3*dE)
+plt.ylabel('Delta energy (meV)')
+plt.twinx()
+plt.plot(mono_turn, ls='', marker='.', c='C1')
+plt.ylabel('mono going up')
+plt.title(f'run {runNB} p{proposalNB}')
+```
+
+%% Cell type:code id: tags:
+
+``` python
+ds['mono_turn'] = xr.DataArray(np.gradient(savgol_filter(ds['nrj'], 9, 1))>0, dims=['trainId'])
+```
+
+%% Cell type:code id: tags:
+
+``` python
+ds
+```
+
+%% Cell type:code id: tags:
+
+``` python
+subds = ds#.isel({'trainId':slice(1600)})
+
+Eshift = 0.05
+shift_subds = subds
+shift_subds['nrj'] = subds['nrj'] + Eshift
+pxas3a = tbr.xas(shift_subds.where((subds['magnet'] > 2)*subds['mono_turn'], drop=True),
+                bins=nrj_bins_edges, plot=False, Itkey=mcp)
+nxas3a = tbr.xas(shift_subds.where((subds['magnet'] < -2)*subds['mono_turn'], drop=True),
+                bins=nrj_bins_edges, plot=False, Itkey=mcp)
+
+shift_subds['nrj'] = subds['nrj'] - Eshift
+pxas3b = tbr.xas(shift_subds.where((subds['magnet'] > 2)*np.logical_not(subds['mono_turn']), drop=True),
+                bins=nrj_bins_edges, plot=False, Itkey=mcp)
+nxas3b = tbr.xas(shift_subds.where((subds['magnet'] < -2)*np.logical_not(subds['mono_turn']), drop=True),
+                bins=nrj_bins_edges, plot=False, Itkey=mcp)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+plt.figure()
+#plt.errorbar(pxas['nrj'], pxas['muA'], pxas['sterrA'])
+plt.errorbar(pxas3a['nrj'], pxas3a['muA'], pxas3a['sterrA'], c='C0', ls='-', label='+H, mono up')
+plt.errorbar(pxas3b['nrj'], pxas3b['muA'], pxas3b['sterrA'], c='C0', ls='--',
+             label=f'+H, mono down, dE = {-1e3*Eshift} meV')
+
+#plt.errorbar(nxas['nrj'], nxas['muA'], nxas['sterrA'])
+plt.errorbar(nxas3a['nrj'], nxas3a['muA'], nxas3a['sterrA'], c='C1', ls='-', label='-H, mono up')
+plt.errorbar(nxas3b['nrj'], nxas3b['muA'], nxas3b['sterrA'], c='C1', ls='--',
+             label=f'-H, mono down, dE = {-1e3*Eshift} meV')
+
+plt.xlabel('Energy (eV)')
+plt.ylabel('XAS (arb. units)')
+plt.title(f'run {runNB} p{proposalNB}')
+plt.xlim([707, 709])
+plt.ylim([3, 4.5])
+plt.legend()
+```
+
+%% Cell type:code id: tags:
+
+``` python
+plt.figure()
+plt.title(f'run {runNB} p{proposalNB}')
+plt.xlabel('Energy (eV)')
+plt.ylabel('XAS (mono up) - XAS (mono down) (arb. units)')
+plt.plot(pxas3a['nrj'], pxas3a['muA'] - pxas3b['muA'])
+plt.plot(nxas3a['nrj'], nxas3a['muA'] - nxas3b['muA'])
+plt.ylim([-0.1, 0.18])
+```
+
+%% Cell type:code id: tags:
+
+``` python
+```

src/toolbox_scs/base/__init__.py

+from . import knife_edge as knife_edge_module
+from .knife_edge import *
+
+
+__all__ = knife_edge_module.__all__