.gitattributes

+doc/changelog.rst merge=union

.gitignore

+.ipynb*
+src/*.egg*
+*.pyc
+*__pycache__*
+tmp/

.gitlab-ci.yml

+# The Docker image that will be used to build your app
+image: sphinxdoc/sphinx
+# Functions that should be executed before the build script is run
+before_script: []
+pages:
+  script:
+    - apt-get update
+    - apt-get install -y pandoc
+    - pip3 install sphinx-autoapi
+    - pip3 install nbsphinx
+    - pip3 install pydata-sphinx-theme
+    - sphinx-build -b html doc public
+  pages: True
+  artifacts:
+    paths:
+      # The folder that contains the files to be exposed at the Page URL
+      - public
+  rules:
+    # This ensures that only pushes to the default branch will trigger
+    # a pages deploy
+    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH

Laser_utils.py

-def positionToDelay(pos, origin=0, invert = False, reflections=1):
-    ''' converts a motor position in mm into optical delay in picosecond
-        Inputs:
-            pos: array-like delay stage motor position
-            origin: motor position of time zero in mm
-            invert: bool, inverts the sign of delay if True
-            reflections: number of bounces in the delay stage
-
-        Output:
-            delay in picosecond
-    '''
-    c_ = 299792458 *1e-9 # speed of light in mm/ps
-    x = -1 if invert else 1
-    return 2*reflections*(pos-origin)*x/c_
-    
-def degToRelPower(x, theta0=0):
-    ''' converts a half-wave plate position in degrees into relative power
-        between 0 and 1.
-        Inputs:
-            x: array-like positions of half-wave plate, in degrees
-            theta0: position for which relative power is zero
-            
-        Output:
-            array-like relative power
-    '''
-    return np.sin(2*(x-theta0)*np.pi/180)**2

README.rst

+Documentation
+#############
+
+Online documentation can be found `here <https://scs.pages.xfel.eu/toolbox/>`_.

VERSION

+1.7.0

__init__.py

-from ToolBox.Load import *
-from ToolBox.xgm import *
-from ToolBox.XAS import *
-from ToolBox.knife_edge import *
-from ToolBox.Laser_utils import *
-from ToolBox.DSSC import DSSC
-from ToolBox.azimuthal_integrator import *
-from ToolBox.DSSC1module import *
-from ToolBox.bunch_pattern import *
-from ToolBox.FastCCD import *

azimuthal_integrator.py

-import numpy as np
-
-class azimuthal_integrator(object):
-    def __init__(self, imageshape, center, polar_range, dr=2, aspect=204/236):
-        '''
-        Create a reusable integrator for repeated azimuthal integration of similar
-        images. Calculates array indices for a given parameter set that allows
-        fast recalculation.
-        
-        Parameters
-        ==========
-        imageshape : tuple of ints
-            The shape of the images to be integrated over.
-            
-        center : tuple of ints
-            center coordinates in pixels
-        
-        polar_range : tuple of ints
-            start and stop polar angle (in degrees) to restrict integration to wedges
-        
-        dr : int, default 2
-            radial width of the integration slices. Takes non-square DSSC pixels into account.
-        
-        aspect: float, default 204/236 for DSSC
-            aspect ratio of the pixel pitch
-        
-        Returns
-        =======
-        ai : azimuthal_integrator instance
-            Instance can directly be called with image data:
-            > az_intensity = ai(image)
-            radial distances and the polar mask are accessible as attributes:
-            > ai.distance
-            > ai.polar_mask
-        '''
-        self.shape = imageshape
-        cx, cy = center
-        print(f'azimuthal center: {center}')
-        sx, sy = imageshape
-        xcoord, ycoord = np.ogrid[:sx, :sy]
-        xcoord -= cx
-        ycoord -= cy
-
-        # distance from center, hexagonal pixel shape taken into account
-        dist_array = np.hypot(xcoord * aspect, ycoord)
-
-        # array of polar angles
-        if np.abs(polar_range[1]-polar_range[0]) > 180:
-            raise ValueError('Integration angle too wide, should be within 180 degrees')
-
-        if np.abs(polar_range[1]-polar_range[0]) < 1e-6:
-            raise ValueError('Integration angle too narrow')
-
-        tmin, tmax = np.deg2rad(np.sort(polar_range)) % np.pi
-        polar_array = np.arctan2(xcoord, ycoord)
-        polar_array = np.mod(polar_array, np.pi)
-        self.polar_mask = (polar_array > tmin) * (polar_array < tmax)
-
-        self.maxdist = max(sx  - cx, sy  - cy)
-
-        ix, iy = np.indices(dimensions=(sx, sy))
-        self.index_array = np.ravel_multi_index((ix, iy), (sx, sy))
-
-        self.distance = np.array([])
-        self.flat_indices = []
-        for dist in range(dr, self.maxdist, dr):
-            ring_mask = self.polar_mask * (dist_array >= (dist - dr)) * (dist_array < dist)
-            self.flat_indices.append(self.index_array[ring_mask])
-            self.distance = np.append(self.distance, dist)
-    
-    def __call__(self, image):
-        assert self.shape == image.shape, 'image shape does not match'
-        image_flat = image.flatten()
-        return np.array([np.nansum(image_flat[indices]) for indices in self.flat_indices])

doc/BOZ.rst

+BOZ: Beam-Splitting Off-axis Zone plate analysis
+------------------------------------------------
+
+The BOZ analysis consists of 4 notebooks and a script. The first notebook
+:doc:`BOZ analysis part I.a Correction determination <BOZ analysis part I.a Correction determination>`
+is used to determine all the necessary correction, that is the flat field
+correction from the zone plate optics and the non-linearity correction from the
+DSSC gain. The inputs are a dark run and a run with X-rays on three broken or
+empty membranes. For the latter, an alternative is to use pre-edge data on an
+actual sample. The result is a JSON file that contains the flat field and
+non-linearity correction as well as the parameters used for their determination
+such that this can be reproduced and investigated in case of issues. The
+determination of the flat field correction is rather quick, few minutes and is
+the most important correction for the change in XAS computed from the -1st and
++1st order. For quick correction of the online preview one can bypass the
+non-linearity calculation by taking the JSON file as soon as it appears.
+The determination of the non-linearity correction is a lot longer and can take
+some 2 to 8 hours depending on the number of pulses in the
+train. For this reason, the computation can also be done on GPUs in 30min
+instead. A GPU notebook adapted for CHEM experiment with liquid jet and 
+normalization implement for S K-edge is available at
+:doc:`OnlineGPU BOZ analysis part I.a Correction determination S K-egde <OnlineGPU BOZ analysis part I.a Correction determination S K-egde>`.
+
+The other option is to use a script
+that can be downloaded from :download:`scripts/boz_parameters_job.sh` and
+reads as:
+
+.. literalinclude:: scripts/boz_parameters_job.sh
+   :language: bash
+   :linenos:
+
+It uses the first notebook and is launched via slurm:
+
+``sbatch ./boz_parameters_job.sh -p 2937 -d 615 -r 614 -g 3``
+
+where 2937 is the proposal run number, where 615 is the dark run number,
+614 is the run on 3 broken membranes and 3 is
+the DSSC gain in photon per bin. The proposal run number is defined inside the
+script file.
+
+The second notebook 
+:doc:`BOZ analysis part I.b Correction validation <BOZ analysis part I.b Correction validation>` can be used to check how well the calculated correction still
+work on a characterization run recorded later, i.e. on 3 broken membrane or empty membranes.
+
+The third notebook
+:doc:`BOZ analysis part II.1 Small data <BOZ analysis part II.1 Small data>`
+then use the JSON correction file to load all needed corrections and
+process an run, saving the rois extracted DSSC as well as aligning them to
+photon energy and delay stage in a small data h5 file.
+
+That small data h5 file can then be loaded and the data binned to compute a
+spectrum or a time resolved XAS scan using the fourth and final notebook
+:doc:`BOZ analysis part II.2 Binning <BOZ analysis part II.2 Binning>`
+
+

doc/DSSC.rst

+DSSC
+----
+
+DSSC data binning
+#################
+
+In scattering experiment one typically wants to bin DSSC image data versus
+time delay between pump and probe or versus photon energy. After this first
+data reduction steps, one can do azimuthal integration on a much smaller
+amount of data.
+
+The DSSC data binning procedure is based on the notebook
+:doc:`Dask DSSC module binning <Dask DSSC module binning>`. It performs
+DSSC data binning against a coordinate specified by *xaxis* which can
+be *nrj* for the photon energy, *delay* in which case the delay stage
+position will be converted in picoseconds and corrected but the BAM, or
+another slow data channel. Specific pulse pattern can be defined, such as:
+
+.. code:: python
+ 
+    ['pumped', 'unpumped']
+
+which will be repeated. XGM data will also be binned similarly to the DSSC
+data.
+
+Since this data reduction step can be quite time consuming for large datasets,
+it is recommended to launch the notebook via a SLURM script. The script can be
+downloaded from :download:`scripts/bin_dssc_module_job.sh` and reads as:
+
+.. literalinclude:: scripts/bin_dssc_module_job.sh
+   :language: bash
+   :linenos:
+
+It is launched with the following:
+
+.. code:: bash
+    
+    sbatch ./bin_dssc_module_job.sh -p 2719 -d 180 -r 179 -m 0 -x delay -b 0.1
+    sbatch ./bin_dssc_module_job.sh -p 2719 -d 180 -r 179 -m 1 -x delay -b 0.1
+    sbatch ./bin_dssc_module_job.sh -p 2719 -d 180 -r 179 -m 2 -x delay -b 0.1
+    sbatch ./bin_dssc_module_job.sh -p 2719 -d 180 -r 179 -m 3 -x delay -b 0.1
+
+where 2719 is the proposal number, 180 is the dark run number, 179 is the run
+nummber and 0, 1, 2 and 3 are the 4 module group, each job processing a set of
+4 DSSC module, delay is the bin axis and 0.1 is the bin width.
+
+The result will be 16 \*.h5 files, one per module, saved in the folder specified
+in the script, a copy of which can be found in the *scripts* folder in the
+toolbox source. This files can then be loaded and combined with:
+
+.. code:: python
+   
+    import xarray as xr
+    data = xr.open_mfdataset(path + '/*.h5', parallel=True, join='inner')
+
+
+DSSC azimuthal integration
+##########################
+
+Azimuthal integration can be performed with pyFAI_ which can utilize the
+hexagonal pixel shape information from the DSSC geometry to split
+the intensity in a pixel in the bins covered by it. An example notebook
+:doc:`Azimuthal integration of DSSC with pyFAI.ipynb <Azimuthal integration of DSSC with pyFAI>` is available.
+
+A second example notebook
+:doc:`DSSC scattering time-delay.ipynb <DSSC scattering time-delay>`
+demonstrates how to:
+
+- refine the geometry such that the scattering pattern is centered before
+  azimuthal integration
+
+- perform azimuthal integration on a time delay
+  dataset with ``xr.apply_ufunc`` for multiprocessing.
+
+- plot a two-dimensional map of the scattering change as function of
+  scattering vector and time delay
+
+- integrate certain scattering vector range and plot a time trace
+
+DSSC fine timing
+################
+
+When DSSC is reused after a period of inactivity or when the DSSC gain setting
+use a different operation frequency the DSSC fine trigger delay needs to be
+checked. To analysis runs recorded with different fine delay, one can use
+the notebook :doc:`DSSC fine delay with SCS toolbox.ipynb <DSSC fine delay with SCS toolbox>`.
+
+DSSC quadrant geometry
+######################
+
+To check or refined the DSSC geometry or quadrants position, the following
+notebook can be used :doc:`DSSC create geometry.ipynb <DSSC create geometry>`.
+
+Legacy DSSC binning procedure
+#############################
+
+Most of the functions within toolbox_scs.detectors can be accessed directly. This is useful during development, or when working in a non-standardized way, which is often neccessary during data evaluation. For frequent routines there is the possibility to use dssc objects that guarantee consistent data structure, and reduce the amount of recurring code within the notebook.
+
+* bin data using toolbox_scs.tbdet -> *to be documented*.
+* :doc:`bin data using the DSSCBinner <dssc/DSSCBinner>`.
+* post processing, data analysis -> *to be documented*
+
+.. _pyFAI: https://pyfai.readthedocs.io