.gitattributes

+doc/changelog.rst merge=union

.gitignore

+.ipynb*
+src/*.egg*
+*.pyc
+*__pycache__*
+tmp/

.gitlab-ci.yml

+# The Docker image that will be used to build your app
+image: sphinxdoc/sphinx
+# Functions that should be executed before the build script is run
+before_script: []
+pages:
+  script:
+    - apt-get update
+    - apt-get install -y pandoc
+    - pip3 install sphinx-autoapi
+    - pip3 install nbsphinx
+    - pip3 install pydata-sphinx-theme
+    - sphinx-build -b html doc public
+  pages: True
+  artifacts:
+    paths:
+      # The folder that contains the files to be exposed at the Page URL
+      - public
+  rules:
+    # This ensures that only pushes to the default branch will trigger
+    # a pages deploy
+    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH

Load.py

-# -*- coding: utf-8 -*-
-""" Toolbox for SCS.
-
-    Various utilities function to quickly process data measured at the SCS instruments.
-
-    Copyright (2019) SCS Team.
-"""
-import numpy as np
-from karabo_data import RunDirectory
-import xarray as xr
-
-mnemonics = {
-    # Machine
-    "sase3": {'source':'SCS_RR_UTC/MDL/BUNCH_DECODER',
-              'key':'sase3.pulseIds.value',
-              'dim':['bunchId']},
-    "sase2": {'source':'SCS_RR_UTC/MDL/BUNCH_DECODER',
-              'key':'sase2.pulseIds.value',
-              'dim':['bunchId']},
-    "sase1": {'source':'SCS_RR_UTC/MDL/BUNCH_DECODER',
-              'key':'sase1.pulseIds.value',
-              'dim':['bunchId']},
-    "maindump": {'source':'SCS_RR_UTC/MDL/BUNCH_DECODER',
-                 'key':'maindump.pulseIds.value',
-                 'dim':['bunchId']},
-    "bunchpattern": {'source':'SCS_RR_UTC/TSYS/TIMESERVER',
-                     'key':'readBunchPatternTable.value',
-                     'dim':None},
-    "npulses_sase3": {'source':'SCS_RR_UTC/MDL/BUNCH_DECODER',
-                      'key':'sase3.nPulses.value',
-                      'dim':None},
-    "npulses_sase1": {'source':'SCS_RR_UTC/MDL/BUNCH_DECODER',
-                      'key':'sase1.nPulses.value',
-                      'dim':None},
-    # SA3
-    "nrj": {'source':'SA3_XTD10_MONO/MDL/PHOTON_ENERGY',
-            'key':'actualEnergy.value',
-            'dim':None},
-
-    "M2BEND": {'source': 'SA3_XTD10_MIRR-2/MOTOR/BENDER',
-               'key': 'actualPosition.value',
-               'dim':None},
-    "VSLIT": {'source':'SA3_XTD10_VSLIT/MDL/BLADE',
-              'key':'actualGap.value',
-              'dim':None},
-    "ESLIT": {'source':'SCS_XTD10_ESLIT/MDL/MAIN',
-              'key':'actualGap.value',
-              'dim':None},
-    "HSLIT": {'source':'SCS_XTD10_HSLIT/MDL/BLADE',
-              'key':'actualGap.value',
-              'dim':None},
-    "transmission": {'source':'SA3_XTD10_GATT/MDL/GATT_TRANSMISSION_MONITOR',
-                     'key':'Estimated_Tr.value',
-                     'dim':None},
-    "GATT_pressure": {'source':'P_GATT',
-                      'key':'value.value',
-                      'dim':None},
-
-    # XGMs
-    "SA3_XGM": {'source':'SA3_XTD10_XGM/XGM/DOOCS:output',
-                'key':'data.intensityTD',
-                'dim':['XGMbunchId']},
-    "SA3_XGM_SLOW": {'source':'SA3_XTD10_XGM/XGM/DOOCS',
-                     'key':'pulseEnergy.photonFlux.value',
-                     'dim':None},
-    "SCS_XGM": {'source':'SCS_BLU_XGM/XGM/DOOCS:output',
-                'key':'data.intensityTD',
-                'dim':['XGMbunchId']},
-    "SCS_XGM_SLOW": {'source':'SCS_BLU_XGM/XGM/DOOCS',
-                     'key':'pulseEnergy.photonFlux.value',
-                     'dim':None},
-
-    # KBS
-    "HFM_CAPB": {'source':'SCS_KBS_HFM/ASENS/CAPB',
-                 'key':'value.value',
-                 'dim':None},
-    "HFM_CAPF": {'source':'SCS_KBS_HFM/ASENS/CAPF',
-                 'key':'value.value',
-                 'dim':None},
-    "HFM_CAPM": {'source':'SCS_KBS_HFM/ASENS/CAPM',
-                 'key':'value.value',
-                 'dim':None},
-    "HFM_BENDERB": {'source':'SCS_KBS_HFM/MOTOR/BENDERB',
-                    'key':'encoderPosition.value',
-                    'dim':None},
-    "HFM_BENDERF": {'source':'SCS_KBS_HFM/MOTOR/BENDERF',
-                    'key':'encoderPosition.value',
-                    'dim':None},
-    "VFM_CAPB": {'source':'SCS_KBS_VFM/ASENS/CAPB',
-                 'key':'value.value',
-                 'dim':None},
-    "VFM_CAPF": {'source':'SCS_KBS_VFM/ASENS/CAPF',
-                 'key':'value.value',
-                 'dim':None},
-    "VFM_CAPM": {'source':'SCS_KBS_VFM/ASENS/CAPM',
-                 'key':'value.value',
-                 'dim':None},
-    "VFM_BENDERB": {'source':'SCS_KBS_VFM/MOTOR/BENDERB',
-                    'key':'encoderPosition.value',
-                    'dim':None},
-    "VFM_BENDERF": {'source':'SCS_KBS_VFM/MOTOR/BENDERF',
-                    'key':'encoderPosition.value',
-                    'dim':None},
-
-    # FFT
-    "scannerX": {'source':'SCS_CDIFFT_SAM/LMOTOR/SCANNERX',
-                 'key':'actualPosition.value',
-                 'dim':None},
-    "scannerY": {'source':'SCS_CDIFFT_SAM/MOTOR/SCANNERY',
-                 'key':'actualPosition.value',
-                 'dim':None},
-    "scannerY_enc": {'source':'SCS_CDIFFT_SAM/ENC/SCANNERY',
-                     'key':'value.value',
-                     'dim':None},
-    "SAM-Z": {'source':'SCS_CDIFFT_MOV/ENC/SAM_Z',
-              'key':'value.value',
-              'dim':None},
-    "magnet": {'source':'SCS_CDIFFT_MAG/SUPPLY/CURRENT',
-               'key':'actual_current.value',
-               'dim':None},
-
-    # FastCCD
-    "fastccd": {'source':'SCS_CDIDET_FCCD2M/DAQ/FCCD:daqOutput',
-                'key':'data.image.pixels',
-                'dim':['x', 'y']},
-
-    # TIM
-    "MCP1apd": {'source':'SCS_UTC1_ADQ/ADC/1:network',
-                'key':'digitizers.channel_1_D.apd.pulseIntegral',
-                'dim':['apdId']},
-    "MCP1raw": {'source':'SCS_UTC1_ADQ/ADC/1:network',
-                'key':'digitizers.channel_1_D.raw.samples',
-                'dim':['samplesId']},
-    "MCP2apd": {'source':'SCS_UTC1_ADQ/ADC/1:network',
-                'key':'digitizers.channel_1_C.apd.pulseIntegral',
-                'dim':['apdId']},
-    "MCP2raw": {'source':'SCS_UTC1_ADQ/ADC/1:network',
-                'key':'digitizers.channel_1_D.raw.samples',
-                'dim':['samplesId']},
-    "MCP3apd": {'source':'SCS_UTC1_ADQ/ADC/1:network',
-                'key':'digitizers.channel_1_B.apd.pulseIntegral',
-                'dim':['apdId']},
-    "MCP3raw": {'source':'SCS_UTC1_ADQ/ADC/1:network',
-                'key':'digitizers.channel_1_D.raw.samples',
-                'dim':['samplesId']},
-    "MCP4apd": {'source':'SCS_UTC1_ADQ/ADC/1:network',
-                'key':'digitizers.channel_1_A.apd.pulseIntegral',
-                'dim':['apdId']},
-    "MCP4raw": {'source':'SCS_UTC1_ADQ/ADC/1:network',
-                'key':'digitizers.channel_1_D.raw.samples',
-                'dim': ['samplesId']},
-    
-    # KARABACON
-    "KARABACON": {'source':'SCS_DAQ_SCAN/MDL/KARABACON',
-                    'key': 'actualStep.value',
-                    'dim': None}
-}
-
-def load(fields, runNB, proposalNB, semesterNB, topic='SCS', display=False, validate=False):
-    """ Load a run and extract the data. Output is an xarray with aligned trainIds
-
-        Inputs:
-            fields: list of mnemonic strings to load specific data such as "fastccd", "SCS_XGM",
-                or dictionnaries defining a custom mnemonic such as
-                {"extra": {'SCS_CDIFFT_MAG/SUPPLY/CURRENT', 'actual_current.value', None}}
-            runNB: run number as integer
-            proposalNB: string of the proposal number
-            semesterNB: string of the semester number where the proposal data are saved
-            topic: string of the topic
-            display: boolean, whether to show the run.info or not
-            validate: boolean, whether to run karabo-data-validate or not
-
-        Outputs:
-            res: an xarray DataSet with aligned trainIds
-    """
-
-    runFolder = '/gpfs/exfel/exp/{}/{}/{}/raw/r{:04d}/'.format(topic, semesterNB, proposalNB, runNB)
-    run = RunDirectory(runFolder)
-
-    if validate:
-        get_ipython().system('karabo-data-validate ' + runFolder)
-    if display:
-        run.info()
-
-    keys = []
-    vals = []
-
-    # always load pulse pattern infos
-    fields += ["sase1", "sase3", "npulses_sase3", "npulses_sase1"]
-
-    for f in fields:
-
-        if type(f) == dict:
-            # extracting mnemomic defined on the spot
-            if len(f.keys()) > 1:
-                print('Loading only one "on-the-spot" mnemonic at a time, skipping all others !')
-            k = list(f.keys())[0]
-            v = f[k]
-        else:
-            # extracting mnemomic from the table
-            if f in mnemonics:
-                v = mnemonics[f]
-                k = f
-            else:
-                print('Unknow mnemonic "{}". Skipping!'.format(f))
-
-        if k in keys:
-            continue # already loaded, skip
-
-        if display:
-            print('Loading {}'.format(k))
-
-        if v['source'] not in run.all_sources:
-            print('Source {} not found in run. Skipping!'.format(v['source']))
-            continue
-
-        vals.append(run.get_array(v['source'], v['key'], extra_dims=v['dim']))
-
-        keys.append(k)
-
-    aligned_vals = xr.align(*vals, join='inner')
-    result = dict(zip(keys, aligned_vals))
-    result = xr.Dataset(result)
-    result.attrs['run'] = run
-    return result

README.rst

+Documentation
+#############
+
+Online documentation can be found `here <https://scs.pages.xfel.eu/toolbox/>`_.

VERSION

+1.7.0

__init__.py

-from ToolBox.Load import *
-from ToolBox.xgm import *

doc/BOZ.rst

+BOZ: Beam-Splitting Off-axis Zone plate analysis
+------------------------------------------------
+
+The BOZ analysis consists of 4 notebooks and a script. The first notebook
+:doc:`BOZ analysis part I.a Correction determination <BOZ analysis part I.a Correction determination>`
+is used to determine all the necessary correction, that is the flat field
+correction from the zone plate optics and the non-linearity correction from the
+DSSC gain. The inputs are a dark run and a run with X-rays on three broken or
+empty membranes. For the latter, an alternative is to use pre-edge data on an
+actual sample. The result is a JSON file that contains the flat field and
+non-linearity correction as well as the parameters used for their determination
+such that this can be reproduced and investigated in case of issues. The
+determination of the flat field correction is rather quick, few minutes and is
+the most important correction for the change in XAS computed from the -1st and
++1st order. For quick correction of the online preview one can bypass the
+non-linearity calculation by taking the JSON file as soon as it appears.
+The determination of the non-linearity correction is a lot longer and can take
+some 2 to 8 hours depending on the number of pulses in the
+train. For this reason, the computation can also be done on GPUs in 30min
+instead. A GPU notebook adapted for CHEM experiment with liquid jet and 
+normalization implement for S K-edge is available at
+:doc:`OnlineGPU BOZ analysis part I.a Correction determination S K-egde <OnlineGPU BOZ analysis part I.a Correction determination S K-egde>`.
+
+The other option is to use a script
+that can be downloaded from :download:`scripts/boz_parameters_job.sh` and
+reads as:
+
+.. literalinclude:: scripts/boz_parameters_job.sh
+   :language: bash
+   :linenos:
+
+It uses the first notebook and is launched via slurm:
+
+``sbatch ./boz_parameters_job.sh -p 2937 -d 615 -r 614 -g 3``
+
+where 2937 is the proposal run number, where 615 is the dark run number,
+614 is the run on 3 broken membranes and 3 is
+the DSSC gain in photon per bin. The proposal run number is defined inside the
+script file.
+
+The second notebook 
+:doc:`BOZ analysis part I.b Correction validation <BOZ analysis part I.b Correction validation>` can be used to check how well the calculated correction still
+work on a characterization run recorded later, i.e. on 3 broken membrane or empty membranes.
+
+The third notebook
+:doc:`BOZ analysis part II.1 Small data <BOZ analysis part II.1 Small data>`
+then use the JSON correction file to load all needed corrections and
+process an run, saving the rois extracted DSSC as well as aligning them to
+photon energy and delay stage in a small data h5 file.
+
+That small data h5 file can then be loaded and the data binned to compute a
+spectrum or a time resolved XAS scan using the fourth and final notebook
+:doc:`BOZ analysis part II.2 Binning <BOZ analysis part II.2 Binning>`
+
+

doc/DSSC.rst

+DSSC
+----
+
+DSSC data binning
+#################
+
+In scattering experiment one typically wants to bin DSSC image data versus
+time delay between pump and probe or versus photon energy. After this first
+data reduction steps, one can do azimuthal integration on a much smaller
+amount of data.
+
+The DSSC data binning procedure is based on the notebook
+:doc:`Dask DSSC module binning <Dask DSSC module binning>`. It performs
+DSSC data binning against a coordinate specified by *xaxis* which can
+be *nrj* for the photon energy, *delay* in which case the delay stage
+position will be converted in picoseconds and corrected but the BAM, or
+another slow data channel. Specific pulse pattern can be defined, such as:
+
+.. code:: python
+ 
+    ['pumped', 'unpumped']
+
+which will be repeated. XGM data will also be binned similarly to the DSSC
+data.
+
+Since this data reduction step can be quite time consuming for large datasets,
+it is recommended to launch the notebook via a SLURM script. The script can be
+downloaded from :download:`scripts/bin_dssc_module_job.sh` and reads as:
+
+.. literalinclude:: scripts/bin_dssc_module_job.sh
+   :language: bash
+   :linenos:
+
+It is launched with the following:
+
+.. code:: bash
+    
+    sbatch ./bin_dssc_module_job.sh -p 2719 -d 180 -r 179 -m 0 -x delay -b 0.1
+    sbatch ./bin_dssc_module_job.sh -p 2719 -d 180 -r 179 -m 1 -x delay -b 0.1
+    sbatch ./bin_dssc_module_job.sh -p 2719 -d 180 -r 179 -m 2 -x delay -b 0.1
+    sbatch ./bin_dssc_module_job.sh -p 2719 -d 180 -r 179 -m 3 -x delay -b 0.1
+
+where 2719 is the proposal number, 180 is the dark run number, 179 is the run
+nummber and 0, 1, 2 and 3 are the 4 module group, each job processing a set of
+4 DSSC module, delay is the bin axis and 0.1 is the bin width.
+
+The result will be 16 \*.h5 files, one per module, saved in the folder specified
+in the script, a copy of which can be found in the *scripts* folder in the
+toolbox source. This files can then be loaded and combined with:
+
+.. code:: python
+   
+    import xarray as xr
+    data = xr.open_mfdataset(path + '/*.h5', parallel=True, join='inner')
+
+
+DSSC azimuthal integration
+##########################
+
+Azimuthal integration can be performed with pyFAI_ which can utilize the
+hexagonal pixel shape information from the DSSC geometry to split
+the intensity in a pixel in the bins covered by it. An example notebook
+:doc:`Azimuthal integration of DSSC with pyFAI.ipynb <Azimuthal integration of DSSC with pyFAI>` is available.
+
+A second example notebook
+:doc:`DSSC scattering time-delay.ipynb <DSSC scattering time-delay>`
+demonstrates how to:
+
+- refine the geometry such that the scattering pattern is centered before
+  azimuthal integration
+
+- perform azimuthal integration on a time delay
+  dataset with ``xr.apply_ufunc`` for multiprocessing.
+
+- plot a two-dimensional map of the scattering change as function of
+  scattering vector and time delay
+
+- integrate certain scattering vector range and plot a time trace
+
+DSSC fine timing
+################
+
+When DSSC is reused after a period of inactivity or when the DSSC gain setting
+use a different operation frequency the DSSC fine trigger delay needs to be
+checked. To analysis runs recorded with different fine delay, one can use
+the notebook :doc:`DSSC fine delay with SCS toolbox.ipynb <DSSC fine delay with SCS toolbox>`.
+
+DSSC quadrant geometry
+######################
+
+To check or refined the DSSC geometry or quadrants position, the following
+notebook can be used :doc:`DSSC create geometry.ipynb <DSSC create geometry>`.
+
+Legacy DSSC binning procedure
+#############################
+
+Most of the functions within toolbox_scs.detectors can be accessed directly. This is useful during development, or when working in a non-standardized way, which is often neccessary during data evaluation. For frequent routines there is the possibility to use dssc objects that guarantee consistent data structure, and reduce the amount of recurring code within the notebook.
+
+* bin data using toolbox_scs.tbdet -> *to be documented*.
+* :doc:`bin data using the DSSCBinner <dssc/DSSCBinner>`.
+* post processing, data analysis -> *to be documented*
+
+.. _pyFAI: https://pyfai.readthedocs.io

doc/Dask DSSC module binning.ipynb

+%% Cell type:code id: tags:
+
+``` python
+import numpy as np
+%matplotlib notebook
+import matplotlib.pyplot as plt
+plt.rcParams['figure.constrained_layout.use'] = True
+
+import dask
+print(f'dask: {dask.__version__}')
+import dask.array as da
+dask.config.set({'array.chunk-size': '512MiB'})
+
+import xarray as xr
+```
+
+%% Output
+
+    dask: 2.11.0
+
+%% Cell type:code id: tags:
+
+``` python
+import sys
+print(sys.executable)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+from psutil import virtual_memory
+import gc
+# gc.collect() # run garbage collection to free possible memory
+
+mem = virtual_memory()
+print(f'Physical memory: {mem.total/1024/1024/1024:.0f} Gb')  # total physical memory available
+```
+
+%% Output
+
+    Physical memory: 504 Gb
+
+%% Cell type:code id: tags:
+
+``` python
+import logging
+logging.basicConfig(filename='example.log', level=logging.DEBUG)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+%load_ext autoreload
+
+%autoreload 2
+
+import toolbox_scs as tb
+print(tb.__file__)
+from toolbox_scs.routines.boz import load_dssc_module
+
+from extra_data import open_run
+```
+
+%% Output
+
+    /home/lleguy/notebooks/ToolBox/src/toolbox_scs/__init__.py
+
+%% Cell type:markdown id: tags:
+
+# Parameters
+
+%% Cell type:code id: tags:parameters
+
+``` python
+proposalNB = 2719
+dark_runNB = 180
+runNB = 179
+module_group = 0
+pulse_pattern = ['pumped', 'unpumped']
+xaxis = 'delay' # 'nrj'
+bin_width = 0.1 # [ps]
+path = f'/gpfs/exfel/exp/SCS/202002/p002719/scratch/tests/r{runNB}/'
+```
+
+%% Cell type:code id: tags:
+
+``` python
+moduleNB = list(range(module_group*4, (module_group+1)*4))
+```
+
+%% Cell type:markdown id: tags:
+
+# Processing function
+
+%% Cell type:code id: tags:
+
+``` python
+def process(module):
+    # Load dark
+    arr_dark, tid_dark = load_dssc_module(proposalNB, dark_runNB, module, drop_intra_darks=False)
+    arr_dark = arr_dark.rechunk(('auto', -1, -1, -1))
+    dark_img = arr_dark.mean(axis=0).compute()
+
+    # Load module data
+    arr, tid = load_dssc_module(proposalNB, runNB, module, drop_intra_darks=False)
+    arr = arr.rechunk(('auto', -1, -1, -1))
+
+    # dark and intra dark correction
+    arr = arr - dark_img
+    arr = arr[:, ::2, :, :] - arr[:, 1::2, :, :]
+
+    # Load slow data against which to bin
+    if xaxis == 'delay':
+        run, v = tb.load(proposalNB, runNB, ['PP800_DelayLine', 'BAM1932M', 'SCS_XGM'])
+    else:
+        run, v = tb.load(proposalNB, runNB, [xaxis, 'SCS_XGM'])
+
+    # select part of the run
+    # v = v.isel({'trainId':slice(0,3000)})
+
+    # combine slow and DSSC module data
+    xr_data = xr.DataArray(arr,
+                       coords={'trainId': tid,
+                               'sa3_pId': v['sa3_pId'].values},
+                       dims = ['trainId', 'sa3_pId', 'y', 'x'])
+    xr_data = xr_data.expand_dims(module=[module], axis=2)
+    r = xr.merge([xr_data.to_dataset(name='DSSC'), v], join='inner')
+
+    # calculate bins
+    if xaxis == 'delay':
+        r['delay'] = tb.misc.positionToDelay(r['PP800_DelayLine'])
+        bam = r['BAM1932M'] - r['BAM1932M'].mean()
+        r['bin_delay'] = ((r['delay'] - bam)/bin_width).round()*bin_width
+    else:
+        r['bin_' + xaxis] = (r[xaxis]/bin_width).round()*bin_width
+
+    # add the pulse pattern coordinates
+    Nrepeats = int(len(v['sa3_pId'].values)/len(pulse_pattern))
+    pp = pulse_pattern*Nrepeats
+    pp = np.array(pp)
+    r = r.assign_coords(pp=("sa3_pId", pp))
+
+    # select pattern and bin data
+    bin_data = None
+    for p in np.unique(pp):
+        # slice using non-index coordinates
+        # https://github.com/pydata/xarray/issues/2028
+        sub_r = r.sel(sa3_pId=(r.pp == p))
+
+        # calculate mean on bin, then mean to remove the dimension
+        res = sub_r.groupby('bin_'+xaxis).mean().mean(['sa3_pId'])
+
+        if bin_data is None:
+            bin_data = res
+            bin_data['DSSC'] = res['DSSC'].expand_dims(pp=[p])
+            bin_data['SCS_SA3'] = res['SCS_SA3'].expand_dims(pp=[p])
+        else:
+            bin_data = xr.merge([bin_data,
+                                 res['DSSC'].expand_dims(pp=[p]),
+                                 res['SCS_SA3'].expand_dims(pp=[p])])
+
+    # save the result
+    fname = path + f'run{runNB}-darkrun{dark_runNB}-module{module}.h5'
+    print(fname)
+    bin_data.to_netcdf(fname, format='NETCDF4', engine='h5netcdf')
+```
+
+%% Cell type:markdown id: tags:
+
+# Processing
+
+%% Cell type:code id: tags:
+
+``` python
+for m in moduleNB:
+    process(m)
+```