doc/known_hosts

+exflqr30526,131.169.220.113 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCcPp1A0EeC72+tSXKqfVfIlxAKq3Bp7CA2LfSbYD9hw4SKp6UhGcM8R5eQ5wnoMCoJ4IwqMqA0Mi24K9kgQxpAao6GOh3Yx+YznLtKGZRx3EYUM+9s+fDEzmZvymZF5aBqfwx6j1aUiSKciVbo/DNjU0PxuTjliicZqqOaaPVZPMHJA9GZuvwvfxcaFp8fDWAMSRKr01rihcJZljXLe8ZLBZTYt8i+W561WrgZiGztLxaBMT9o+MbEkHsUor5xBidW6NOH1TkQw3wiVp0vl6/ca9VskMIr3OuWCdce6YqR3GC3wBJMMgUeFxyKr3t0k1b4Voxbcy1dh0ObaFPniBAv

doc/load.rst

+Loading run data
+================
+
+.. toctree::
+
+   Loading_data_in_memory
+
+Short version
+-------------
+
+Loading data in memory is performed as follows:
+
+**Option 1**:
+
+.. code:: python3
+
+    import toolbox_scs as tb
+
+    # optional, check available mnemonics
+    # print(tb.mnemonics)
+    # fields is a list of available mnemonics, representing the data
+    # to be loaded
+
+    fields = ["FastADC4raw", "scannerX"]
+    proposalNr = 2565
+    runNr = 19
+
+    run, data = tb.load(proposalNr, runNr, fields)
+
+run is an extra_data dataCollection and data is an xarray Dataset containing all variables listed in fields. All variables are aligned by train Id.
+
+**Option 2**:
+
+.. code:: python3
+
+    import toolbox_scs as tb
+
+    # get entry for single data source defined in mnemonics
+    proposalNr = 2565
+    runNr = 19
+
+    run = tb.open_run(proposalNr, runNr)
+    data = tb.get_array(run, "scannerX")
+
+run is an extra_data dataCollection and data an xarray dataArray for a single data source.

doc/maintainers.rst

+.. _maintainers:
+
+Maintainers
+~~~~~~~~~~~
+
+Creating a Toolbox environment in the proposal folder
+-----------------------------------------------------
+
+A Toolbox environment can be created by running the following commands
+in Maxwell:
+
+.. code:: shell
+
+    module load exfel exfel-python
+    scs-create-toolbox-env --proposal <PROPOSAL>
+
+where ``<PROPOSAL>`` is the desired proposal number. This will create a Python
+environment and will download and install the Toolbox source code. It will
+result to the creation of the following folders in the path
+``<PROPOSAL_PATH>/scratch``.
+
+.. code-block::
+
+    <PROPOSAL_PATH>/scratch/
+    ├─ checkouts/
+    │  ├─ toolbox_<PROPOSAL>/
+    │  │  ├─ <source code>
+    ├─ envs/
+    │  ├─ toolbox_<PROPOSAL>/
+    │  │  ├─ <Python files>
+
+
+The ``checkouts`` folder contains the Toolbox source codes, correspondingly
+labeled according to the environment identifier, which is the proposal number
+by default. The downloaded code defaults to the **master** version at the
+time when the environment is created.
+
+The ``envs`` folder contains the Python environment with the packages necessary
+to run the Toolbox. It is also correspondingly labeled according to the
+environment identifier, which is the proposal number by default.
+
+.. note::
+
+    One can find the proposal path by running ``findxfel <PROPOSAL>``.
+
+
+It is a good practice to tag the Toolbox version at a given milestone.
+This version can be then supplied to the script as:
+
+.. code:: shell
+
+    scs-create-toolbox-env --proposal <PROPOSAL> --version <VERSION>
+
+
+It might also be helpful to supply an identifier to distinguish environments
+from each other. This can be done by running:
+
+.. code:: shell
+
+    scs-create-toolbox-env --proposal <PROPOSAL> --identifier <IDENTIFIER>
+
+
+The environment would then be identified as ``toolbox_<IDENTIFIER>`` instead
+of ``toolbox_<PROPOSAL>``.
+
+
+Installing additional packages
+------------------------------
+
+In order to install additional packages in a Toolbox environment, one should
+run the following commands:
+
+.. code:: shell
+
+    cd <PROPOSAL_PATH>/scratch/
+    source envs/toolbox_<IDENTIFIER>/bin/activate
+    pip install ...
+
+There's no need to load the ``exfel`` module.
+
+
+Updating the source codes
+--------------------------
+
+Should there be desired changes in the Toolbox codes, may it be bug fixes or
+additional features during beamtime, one can freely modify the source codes in
+the following path: ``<PROPOSAL_PATH>/scratch/checkouts/toolbox_<IDENTIFIER>``.
+The contents of this folder should be a normal git repository. Any changes can
+be easily done (e.g., editing a line of code, checking out a different
+branch, etc.) and such changes are immediately reflected on the environment.

doc/requirements.txt

+sphinx
+sphinx_rtd_theme
+autoapi
+sphinx-autoapi
+nbsphinx
+urllib3<2.0.0

doc/scripts/bin_dssc_module_job.sh

+#!/bin/bash
+#SBATCH -N 1
+#SBATCH --partition=exfel
+#SBATCH --time=12:00:00
+#SBATCH --mail-type=END,FAIL
+#SBATCH --output=logs/%j-%x.out
+
+while getopts ":p:d:r:k:m:x:b:" option
+do
+    case $option in
+        p) PROPOSAL="$OPTARG";;
+        d) DARK="$OPTARG";;
+        r) RUN="$OPTARG";;
+        k) KERNEL="$OPTARG";;
+        m) MODULE_GROUP="$OPTARG";;
+        x) XAXIS="$OPTARG";;
+        b) BINWIDTH="$OPTARG";;
+        \?) echo "Unknown option"
+            exit 1;;
+        :) echo "Missing option for input flag"
+           exit 1;;
+    esac
+done
+
+# Load xfel environment
+source /etc/profile.d/modules.sh
+module load exfel exfel-python
+
+echo processing run $RUN
+PDIR=$(findxfel $PROPOSAL)
+PPROPOSAL="p$(printf '%06d' $PROPOSAL)"
+RDIR="$PDIR/usr/processed_runs/r$(printf '%04d' $RUN)"
+mkdir $RDIR
+
+NB='Dask DSSC module binning.ipynb'
+
+# kernel list can be seen from 'jupyter kernelspec list'
+if [ -z "${KERNEL}" ]; then
+	  KERNEL="toolbox_$PPROPOSAL"
+fi
+
+python -c "import papermill as pm; pm.execute_notebook(\
+  '$NB', \
+  '$RDIR/output$MODULE_GROUP.ipynb', \
+  kernel_name='$KERNEL', \
+  parameters=dict(proposalNB=int('$PROPOSAL'), \
+                  dark_runNB=int('$DARK'), \
+                  runNB=int('$RUN'), \
+                  module_group=int('$MODULE_GROUP'), \
+                  path='$RDIR/', \
+                  xaxis='$XAXIS', \
+                  bin_width=float('$BINWIDTH')))"

doc/scripts/boz_parameters_job.sh

+#!/bin/bash
+#SBATCH -N 1
+#SBATCH --partition=allgpu
+#SBATCH --constraint=V100
+#SBATCH --time=2:00:00
+#SBATCH --mail-type=END,FAIL
+#SBATCH --output=logs/%j-%x.out
+
+ROISTH='1'
+SATLEVEL='500'
+MODULE='15'
+
+while getopts ":p:d:r:k:g:t:s:m:" option
+do
+    case $option in
+        p) PROPOSAL="$OPTARG";;
+        d) DARK="$OPTARG";;
+        r) RUN="$OPTARG";;
+        k) KERNEL="$OPTARG";;
+        g) GAIN="$OPTARG";;
+        t) ROISTH="$OPTARG";;
+        s) SATLEVEL="$OPTARG";;
+        m) MODULE="$OPTARG";;
+        \?) echo "Unknown option"
+            exit 1;;
+        :) echo "Missing option for input flag"
+           exit 1;;
+    esac
+done
+
+# Load xfel environment
+source /etc/profile.d/modules.sh
+module load exfel exfel-python
+
+echo processing run $RUN
+PDIR=$(findxfel $PROPOSAL)
+PPROPOSAL="p$(printf '%06d' $PROPOSAL)"
+RDIR="$PDIR/usr/processed_runs/r$(printf '%04d' $RUN)"
+mkdir $RDIR
+
+NB='BOZ analysis part I.a Correction determination.ipynb'
+
+# kernel list can be seen from 'jupyter kernelspec list'
+if [ -z "${KERNEL}" ]; then
+	KERNEL="toolbox_$PPROPOSAL"
+fi
+
+python -c "import papermill as pm; pm.execute_notebook(\
+  '$NB', \
+  '$RDIR/output.ipynb', \
+  kernel_name='$KERNEL', \
+  parameters=dict(proposal=int('$PROPOSAL'), \
+                  darkrun=int('$DARK'), \
+                  run=int('$RUN'), \
+                  module=int('$MODULE'), \
+                  gain=float('$GAIN'), \
+                  rois_th=float('$ROISTH'), \
+                  sat_level=int('$SATLEVEL')))"

doc/scripts/format_data.py

+import os
+import logging
+import argparse
+
+import numpy as np
+
+import toolbox_scs as tb
+import toolbox_scs.detectors as tbdet
+
+logging.basicConfig(level=logging.INFO)
+log_root = logging.getLogger(__name__)
+
+# -----------------------------------------------------------------------------
+# user input:
+# -----------------------------------------------------------------------------
+run_type = 'static, delay, .....'
+description = 'useful description or comment .....'
+#add xgm data to formatted file if save_xgm_binned was set to True
+metadata = ['binner1', 'binner2', 'xgm_binned'] # ['binner1', 'binner2'] 
+# -----------------------------------------------------------------------------
+
+
+def formatting(run_number, run_folder):
+    log_root.debug("Collect, combine and format files in run folder")
+
+    run_formatted = tbdet.DSSCFormatter(run_folder)
+    run_formatted.combine_files()
+
+    run_formatted.add_dataArray(metadata)
+
+    attrs = {'run_type':run_type,
+             'description':description,
+             'run_number':run_number}
+    run_formatted.add_attributes(attrs)
+
+    run_formatted.save_formatted_data(f'{run_folder}run_{run_number}_formatted.h5')
+    log_root.debug("Formatting finished successfully.")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--run-number', metavar='S',
+                        action='store',
+                        help='run number')
+    parser.add_argument('--run-folder', metavar='S',
+                        action='store',
+                        help='the run folder containing fractional data')
+    args = parser.parse_args()
+
+    formatting(str(args.run_number), str(args.run_folder))

doc/scripts/format_data.sh

+#!/bin/bash
+
+RUN_NR=${1}
+RUN_DIR="../processed_runs/r_${RUN_NR}/"
+
+
+if [ -d $RUN_DIR ]
+    then 
+        echo creating formatted .h5 file for run $RUN_NR in $RUN_DIR
+	source /etc/profile.d/modules.sh
+        module load exfel exfel-python
+
+        python format_data.py --run-number $RUN_NR --run-folder $RUN_DIR
+        #chgrp -R 60002711-part $RUN_DIR
+        chmod -R 777 $RUN_DIR
+    else
+        echo run folder $RUN_DIR does not exist
+        echo please provide a valid run number
+fi

doc/scripts/start_job_single.sh

+#!/bin/bash
+#SBATCH -N 1
+#SBATCH --partition=upex
+#SBATCH --time=00:30:00
+#SBATCH --mail-type=END,FAIL
+#SBATCH --output=../logs/%j-%x.out
+
+RUN=$1
+MODULES=$2
+RUNTYPE=$3
+
+source /etc/profile.d/modules.sh
+module load exfel exfel-python
+
+echo processing modules $MODULES of run $RUN
+python process_data_201007_23h.py --run-number $RUN --module ${MODULES} --runtype $RUNTYPE

doc/scripts/start_processing_all.sh

+#!/bin/bash
+
+RUN=$1
+RUNTYPE=$2
+
+if [ $RUN ] && [ $RUNTYPE ]
+    then
+        echo processing run $RUN
+        source /etc/profile.d/modules.sh
+        module load exfel exfel-python
+        
+        sbatch ./start_job_single.sh $RUN '0 1 2 3' $RUNTYPE
+        sbatch ./start_job_single.sh $RUN '4 5 6 7' $RUNTYPE
+        sbatch ./start_job_single.sh $RUN '8 9 10 11' $RUNTYPE
+        sbatch ./start_job_single.sh $RUN '12 13 14 15' $RUNTYPE
+    else
+        echo please specify a run number and type
+        echo available runtypes:
+        echo energyscan, energyscan_pumped, static, static_IR, delayscan, timescan
+fi

doc/transient reflectivity.rst

+Finding time overlap by transient reflectivity
+----------------------------------------------
+
+Transient reflectivity of the optical laser measured on a large bandgap material pumped by the FEL is often used at SCS to find the time overlap between the two beams. The example notebook 
+
+* :doc:`Transient reflectivity measurement <Transient reflectivity measurement>`
+
+shows how to analyze such data, including correcting the delay by the bunch arrival monitor (BAM).
+

knife_edge.py

-""" Toolbox for SCS.
-
-    Various utilities function to quickly process data measured at the SCS instruments.
-
-    Copyright (2019) SCS Team.
-"""
-import matplotlib.pyplot as plt
-import numpy as np
-from scipy.special import erfc
-from scipy.optimize import curve_fit
-import bisect
-
-def knife_edge(nrun, axisKey='scannerX', signalKey='FastADC4peaks', 
-               axisRange=[None,None], p0=None, full=False, plot=False):
-    ''' Calculates the beam radius at 1/e^2 from a knife-edge scan by fitting with
-        erfc function: f(a,b,u) = a*erfc(u)+b or f(a,b,u) = a*erfc(-u)+b where 
-        u = sqrt(2)*(x-x0)/w0 with w0 the beam radius at 1/e^2 and x0 the beam center.
-        Inputs:
-            nrun: xarray Dataset containing the detector signal and the motor 
-                  position.
-            axisKey: string, key of the axis against which the knife-edge is 
-                  performed.
-            signalKey: string, key of the detector signal.
-            axisRange: list of length 2, minimum and maximum values between which to apply
-                       the fit.
-            p0: list, initial parameters used for the fit: x0, w0, a, b. If None, a beam
-                radius of 100 um is assumed.
-            full: bool: If False, returns the beam radius and standard error. If True,
-                returns the popt, pcov list of parameters and covariance matrix from 
-                curve_fit as well as the fitting function.
-            plot: bool: If True, plots the data and the result of the fit.
-        Outputs:
-            If full is False, ndarray with beam radius at 1/e^2 in mm and standard 
-            error from the fit in mm. If full is True, returns popt and pcov from 
-            curve_fit function.
-    '''
-    def integPowerUp(x, x0, w0, a, b):
-        return a*erfc(-np.sqrt(2)*(x-x0)/w0) + b
-
-    def integPowerDown(x, x0, w0, a, b):
-        return a*erfc(np.sqrt(2)*(x-x0)/w0) + b
-
-    #get the number of pulses per train from the signal source:
-    dim = nrun[signalKey].dims[1]
-    #duplicate motor position values to match signal shape
-    #this is much faster than using nrun.stack()
-    positions = np.repeat(nrun[axisKey].values, 
-                          len(nrun[dim])).astype(nrun[signalKey].dtype)
-    #sort the data to decide which fitting function to use
-    sortIdx = np.argsort(positions)
-    positions = positions[sortIdx]
-    intensities = nrun[signalKey].values.flatten()[sortIdx]
-    
-    if axisRange[0] is None or axisRange[0] < positions[0]:
-        idxMin = 0
-    else:
-        if axisRange[0] >= positions[-1]:
-            raise ValueError('The minimum value of axisRange is too large')
-        idxMin = bisect.bisect(positions, axisRange[0])
-    if axisRange[1] is None or axisRange[1] > positions[-1]:
-        idxMax = None
-    else:
-        if axisRange[1] <= positions[0]:
-            raise ValueError('The maximum value of axisRange is too small')
-        idxMax = bisect.bisect(positions, axisRange[1]) + 1
-    positions = positions[idxMin:idxMax]
-    intensities = intensities[idxMin:idxMax]
-       
-    # estimate a linear slope fitting the data to determine which function to fit
-    slope = np.cov(positions, intensities)[0][1]/np.var(positions) 
-    if slope < 0:
-        func = integPowerDown
-        funcStr = 'a*erfc(np.sqrt(2)*(x-x0)/w0) + b'
-    else:
-        func = integPowerUp
-        funcStr = 'a*erfc(-np.sqrt(2)*(x-x0)/w0) + b'
-    if p0 is None:
-        p0 = [np.mean(positions), 0.1, np.max(intensities)/2, 0]
-    popt, pcov = curve_fit(func, positions, intensities, p0=p0)
-    print('fitting function:', funcStr)
-    print('w0 = (%.1f +/- %.1f) um'%(popt[1]*1e3, pcov[1,1]**0.5*1e3))
-    print('x0 = (%.3f +/- %.3f) mm'%(popt[0], pcov[0,0]**0.5))
-    print('a = %e +/- %e '%(popt[2], pcov[2,2]**0.5))
-    print('b = %e +/- %e '%(popt[3], pcov[3,3]**0.5))
-    
-    if plot:
-        xfit = np.linspace(positions.min(), positions.max(), 1000)
-        yfit = func(xfit, *popt)
-        plt.figure(figsize=(7,4))
-        plt.scatter(positions, intensities, color='C1', label='exp', s=2, alpha=0.01)
-        plt.plot(xfit, yfit, color='C4', 
-                 label=r'fit $\rightarrow$ $w_0=$(%.1f $\pm$ %.1f) $\mu$m'%(popt[1]*1e3, pcov[1,1]**0.5*1e3))
-        leg = plt.legend()
-        for lh in leg.legendHandles: 
-            lh.set_alpha(1)
-        plt.ylabel(signalKey)
-        plt.xlabel(axisKey + ' position [mm]')
-        plt.title(nrun.attrs['runFolder'])
-        plt.tight_layout()
-    if full:
-        return popt, pcov, func
-    else:
-        return np.array([popt[1], pcov[1,1]**0.5])

src/toolbox_scs/base/__init__.py

+from . import knife_edge as knife_edge_module
+from .knife_edge import *
+
+
+__all__ = knife_edge_module.__all__