Merge branch 'feat/new_agipd_ff_localoutput' into 'master'

[AGIPD][FLAT FIELDS] Initial Implementation

See merge request detectors/pycalibration!380

cal_tools/cal_tools/agipdlib.py

@@ -624,7 +624,7 @@ class AgipdCorrections:
         # slopeFF = slopeFFpix/avarege(slopeFFpix)
         # To apply them we have to / not *
         if self.corr_bools.get("xray_corr"):
-            data /= self.xray_cor[module_idx]
+            data /= self.xray_cor[module_idx][cellid, ...]
 
         # use sharedmem raw_data and t0_rgain
         # after calculating it while offset correcting.
@@ -818,7 +818,7 @@ class AgipdCorrections:
             uq, fidxv, cntsv = np.unique(trains, return_index=True,
                                          return_counts=True)
 
-            # Validate calculated CORR INDEX contents by checking 
+            # Validate calculated CORR INDEX contents by checking
             # difference between trainId stored in RAW data and trains from
             train_diff = np.isin(np.array(infile["/INDEX/trainId"]), uq,
                                  invert=True)
@@ -905,12 +905,12 @@ class AgipdCorrections:
         exists of the current AGIPD instances.
 
         Relative gain is derived both from pulse capacitor as well as low
-        intensity flat field data, information from flat field data is 
-        needed to 'calibrate' pulse capacitor data, if there is no 
+        intensity flat field data, information from flat field data is
+        needed to 'calibrate' pulse capacitor data, if there is no
         available FF data, relative gain for High Gain stage is set to 1:
 
-        * Relative gain for High gain stage - from the FF data we get 
-          the relative slopes of a given pixel and memory cells with 
+        * Relative gain for High gain stage - from the FF data we get
+          the relative slopes of a given pixel and memory cells with
           respect to all memory cells and all pixels in the module,
           Please note: Current slopesFF avaialble in calibibration
           constants are created per pixel only, not per memory cell:
@@ -923,9 +923,9 @@ class AgipdCorrections:
           between high and medium gain using slope information from
           fits to the linear part of high and medium gain:
 
-             rfpc_high_medium = m_h/m_m          
+             rfpc_high_medium = m_h/m_m
 
-          where m_h and m_m is the medium gain slope of given memory cells 
+          where m_h and m_m is the medium gain slope of given memory cells
           and pixel and m_h is the high gain slope as above
            rel_gain_medium = rel_high_gain * rfpc_high_medium
 
@@ -954,32 +954,16 @@ class AgipdCorrections:
 
         if self.corr_bools.get("xray_corr"):
             bpixels |= cons_data["BadPixelsFF"].astype(np.uint32)[..., :bpixels.shape[2], None]  # noqa
-            slopesFF = np.squeeze(cons_data["SlopesFF"])
-
+            slopesFF = cons_data["SlopesFF"]
             if len(slopesFF.shape) == 4:
                 slopesFF = slopesFF[..., 0]
-
-            # first 32 cells are known to behave differently so if we can avoid
-            # them
-            # when calculating the mean X-ray derived gain slope for each pixel
-            if slopesFF.shape[2] > 32:
-                xray_cor = np.nanmedian(
-                    slopesFF[..., 32:min(96, self.max_cells)], axis=2)
-            elif slopesFF.shape[2] > 2:
-                xray_cor = np.nanmedian(
-                    slopesFF[..., :min(96, self.max_cells)], axis=2)
-            else:
-                xray_cor = np.squeeze(slopesFF[..., 0])
+            # Memory cell resolved xray_cor correction
+            xray_cor = slopesFF  # (128, 512, mem_cells)
 
             # relative X-ray correction is normalized by the median
             # of all pixels
 
-            # TODO: A check is required to know why it is again divided by 
-            # median. If we have relative slopes in the constants
-            # and (we have!) 
-            # xray cor = (slopeFF/avarege_slopeFF)/avarege_slopeFF.
-            # It didn't not make sense and was removed. 
-            # xray_cor /= np.nanmedian(xray_cor)
+            xray_cor /= np.nanmedian(xray_cor)
 
             self.xray_cor[module_idx][...] = xray_cor.transpose()[...]
 
@@ -1041,13 +1025,19 @@ class AgipdCorrections:
             # ration between HG and MG per pixel per mem cell used
             # for rel gain calculation
             frac_high_med_pix = pc_high_m / pc_med_m
-            # avarage ration between HG and MG as a function of 
+            # avarage ration between HG and MG as a function of
             # mem cell (needed for bls_stripes)
             # TODO: Per pixel would be more optimal correction
             frac_high_med = pc_high_med / pc_med_med
             # calculate additional medium-gain offset
             md_additional_offset = pc_high_l - pc_med_l * pc_high_m / pc_med_m
 
+            # Calculate relative gain. If FF constants are available,
+            # use them for high gain
+            # if not rel_gain is calculated using PC data only
+            # if self.corr_bools.get("xray_corr"):
+            #     rel_gain[..., :self.max_cells, 0] /= xray_corr
+
             # PC data should be 'calibrated with X-ray data,
             # if it is not done, it is better to use 1 instead of bias
             # the results with PC arteffacts.
@@ -1078,17 +1068,35 @@ class AgipdCorrections:
         dname = device.device_name
         cons_data = dict()
         when = dict()
+
         for cname, mdata in const_yaml[dname].items():
             when[cname] = mdata["creation-time"]
             if when[cname]:
-                cf = h5py.File(mdata["file-path"], "r")
-                cons_data[cname] = np.copy(cf[f"{dname}/{cname}/0/data"])
-                cf.close()
+                # This path is only used when testing new flat fields from
+                # file during development: it takes ages to test using all
+                # cells. Consequently, the shape needs to be fixed when less
+                # cells are used.
+                with h5py.File(mdata["file-path"], "r") as cf:
+                    cons_data[cname] = np.copy(cf[f"{dname}/{cname}/0/data"])
+                    shape = cons_data[cname].shape  # (128, 512, mem_cells)
+                    extra_dims = shape[:2] + (self.max_cells-shape[2], )
+
+                    if extra_dims[-1] != 0 and cname == "BadPixelsFF":
+                        extra_temp = np.zeros(extra_dims, dtype=np.int32)
+                        cons_data[cname] = np.concatenate(
+                            (cons_data[cname], extra_temp), axis=2)
+                        print('An extra dimension was added to the constants '
+                              'for the benefit of BadPixelsFF')
+
+                    if extra_dims[-1] != 0 and cname == "SlopesFF":
+                        extra_temp = np.ones(extra_dims, dtype=np.float32)
+                        cons_data[cname] = np.concatenate(
+                            (cons_data[cname], extra_temp), axis=2)
+                        print('An extra dimension was added to the constants '
+                              'for the benefit of SlopesFF')
             else:
                 # Create empty constant using the list elements
-                cons_data[cname] = \
-                    getattr(np, mdata["file-path"][0])(mdata["file-path"][1])
-
+                cons_data[cname] = getattr(np, mdata["file-path"][0])(mdata["file-path"][1])  # noqa
         self.init_constants(cons_data, module_idx)
 
         return when
@@ -1191,7 +1199,7 @@ class AgipdCorrections:
                                                              dtype='f4')
 
             self.mask[module_idx] = sharedmem.empty(constant_shape, dtype='i4')
-            self.xray_cor[module_idx] = sharedmem.empty(constant_shape[2:],
+            self.xray_cor[module_idx] = sharedmem.empty(constant_shape[1:],
                                                         dtype='f4')
 
     def allocate_images(self, shape, n_cores_files):

cal_tools/cal_tools/agipdutils.py

@@ -43,7 +43,7 @@ def assemble_constant_dict(corr_bools, pc_bools, memory_cells, bias_voltage,
     const_dict = {
         "Offset": ["zeros", (128, 512, memory_cells, 3), darkcond],
         "Noise": ["zeros", (128, 512, memory_cells, 3), darkcond],
-        "ThresholdsDark": ["ones", (128, 512, memory_cells, 2), darkcond],
+        "ThresholdsDark": ["ones", (128, 512, memory_cells, 5), darkcond],
         "BadPixelsDark": ["zeros", (128, 512, memory_cells, 3), darkcond],
     }
 

cal_tools/cal_tools/agipdutils_ff.py

+from typing import Any, Dict, List, Optional, Tuple
+
+from iminuit import Minuit
+import numpy as np
+
+from cal_tools.enums import BadPixelsFF
+
+
+def any_in(mask: np.ndarray, bits: int) -> bool:
+    return mask.astype(np.uint) & bits > 0
+
+
+def gaussian(x: np.ndarray, norm: int = 1, mean: int = 0, sigma: int = 1) -> float:  # noqa
+    """
+    Return value of Gaussian function
+
+    :param x: Argument (float of 1D array) of Gaussian function
+    :param norm: Normalization of Gaussian function
+    :param mean: Mean parameter
+    :param sigma: Sigma parameter
+    :return: Value of gaussian function.
+    """
+    return norm * np.exp(-1 / 2 * ((x - mean) / sigma) ** 2) / (sigma * np.sqrt(2 * np.pi))  # noqa
+
+
+def gaussian_sum(x: np.ndarray, ng: int = 4, *p: Tuple[Any]) -> float:
+    """Sum of ng Gaussian functions
+
+    :param x: Argument (float of 1D array) of the function
+    :param ng: Number of Gaussian functions
+    :param p: List of parameters (norm1,mean1,sigma1,norm2,mean2,sigma2,...)
+    """
+    r = 0.
+    for i in range(ng):
+        r += gaussian(x, *p[i * 3:(i + 1) * 3])
+    return r
+
+
+def get_statistical_parameters(x: np.ndarray,
+                               y: np.ndarray,
+                               x_range: List) -> Tuple[np.uint64, np.float64, np.float64, np.ndarray]:  # noqa
+    """Return statistical parameters of selected part of a histogram.
+
+    :param x: Center of bins of the histogram
+    :param y: Value of bins of the histogram
+    :param x_range: x range to be considered
+    :return: Sum of histogram, Mean value, Standard deviation,
+             List of selected bins
+    """
+    # TODO: Check if wq.median works better than mean
+    sel = (x >= x_range[0]) & (x < x_range[1])
+    h_sum = np.sum(y[sel])
+    h_norm = y[sel] / h_sum
+    h_mean = np.sum(h_norm * x[sel])
+    h_sqr = (x[sel] - h_mean) ** 2
+    h_std = np.sqrt(np.sum(h_norm * h_sqr))
+
+    return h_sum, h_mean, h_std, sel
+
+
+def get_starting_parameters(xe: np.ndarray,
+                            ye: np.ndarray,
+                            limits: np.ndarray,
+                            n_peaks: int = 3,
+                            f_lim: int = 2) -> Tuple[Dict[str, Any], List[Tuple]]:  # noqa
+    """
+    Estimate starting parameters for Gaussian fit of several peaks.
+
+    :param xe: Center of bins of the histogram
+    :param ye: Value of bins of the histogram
+    :param limits: Position of each peak ((left1, right1),
+    (left2, right2), ...) to be considered.
+    :param n_peaks: Number of peaks
+    :param f_lim: Limits in units of standard deviation to consider
+    """
+    parameters = {}
+    shapes = []
+    for peak in range(n_peaks):
+        n, m, rms, idx = get_statistical_parameters(xe, ye, limits[peak])
+        limits2 = [m - f_lim * rms, m + f_lim * rms]
+        n, m, rms, idx = get_statistical_parameters(xe, ye, limits2)
+        shapes.append((n, m, rms, idx))
+
+        parameters.update({f'g{peak}sigma': rms,
+                           f'g{peak}n': float(n),
+                           f'g{peak}mean': m})
+    return parameters, shapes
+
+
+def fit_n_peaks(x: np.ndarray,
+                y: np.ndarray,
+                pars: Dict[str, Any],
+                x_range: Tuple[float, float],
+                do_minos: Optional[bool] = False,
+                n_peaks: Optional[int] = 4,
+                fix_d01: Optional[bool] = True) -> Minuit:
+    """
+    Fit histogram with n-Gaussian function.
+
+    :param x: Center of bins of the histogram
+    :param y: Value of bins of the histogram
+    :param pars: Dictionary of initial parameters for fitting
+    :param x_range: x Range to be considered for the fitting
+    :param do_minos: Run Minos if True
+    :param n_peaks: Number of Gaussian peaks to fit (min 2, max 4)
+    :param fix_d01: Fix position of peaks to the distance between noise and
+    first photon peak.
+    :return: minuit object
+    """
+    sel = (x >= x_range[0]) & (x < x_range[1])
+
+    # Square of bin errors
+    yrr2 = np.copy(y[sel])
+    yrr2[yrr2 == 0] = 1  # bins with zero events have error=1
+
+    if fix_d01:
+        pars['fix_g2mean'] = True
+        pars['fix_g3mean'] = True
+
+    if n_peaks < 4:
+        pars['g3n'] = 0
+        pars['fix_g3n'] = True
+        pars['g3sigma'] = 1
+        pars['fix_g3sigma'] = True
+        pars['fix_g3mean'] = True
+
+    if n_peaks < 3:
+        pars['g2n'] = 0
+        pars['fix_g2n'] = True
+        pars['g2sigma'] = 1
+        pars['fix_g2sigma'] = True
+        pars['fix_g2mean'] = True
+
+    def chi2_f(g0n, g0mean, g0sigma,
+               g1n, g1mean, g1sigma,
+               g2n, g2mean, g2sigma,
+               g3n, g3mean, g3sigma, ):
+
+        d01 = (g1mean - g0mean)
+
+        if 'fix_g2mean' in pars and pars['fix_g2mean']:
+            g2mean = g0mean + d01 * 2
+
+        if 'fix_g3mean' in pars and pars['fix_g3mean']:
+            g3mean = g0mean + d01 * 3
+
+        if g3n == 0:
+            n_peaks = 3
+        elif g2n == 0:
+            n_peaks = 2
+        else:
+            n_peaks = 4
+
+        yt = gaussian_sum(x[sel], n_peaks,
+                          g0n, g0mean, g0sigma,
+                          g1n, g1mean, g1sigma,
+                          g2n, g2mean, g2sigma,
+                          g3n, g3mean, g3sigma)
+        return np.nansum((yt - y[sel]) ** 2 / yrr2)
+
+    minuit = Minuit(chi2_f, **pars, pedantic=False)
+    minuit.migrad()
+
+    if do_minos:
+        if minuit.get_fmin().is_valid:
+            minuit.minos()
+
+    return minuit
+
+
+def set_par_limits(pars: Dict[str, Any],
+                   peak_range: np.ndarray,
+                   peak_norm_range: np.ndarray,
+                   peak_width_range: np.ndarray,
+                   n_peaks: Optional[int] = 4):
+    """
+    Set limits on initial fit parameters based on given values
+
+    :param pars: Dictionary of initial fit parameters
+    :param peak_range: Limits on peak positions
+    :param peak_norm_range: Limits on normalization of Gaussian peaks
+    :param peak_width_range: Limits on width of Gaussian peaks
+    :param n_peaks: Number of Gaussian peaks
+    """
+    for peak in range(n_peaks):
+        pars.update({f'limit_g{peak}n': peak_norm_range[peak],
+                     f'limit_g{peak}mean': peak_range[peak],
+                     f'limit_g{peak}sigma': peak_width_range[peak],
+                     })
+
+
+def get_mask(fit_summary: Dict[str, Any],
+             peak_lim: List,
+             d0_lim: List,
+             chi2_lim: List,
+             peak_width_lim: np.array) -> int:
+    """
+    Calculate Bad pixels mask based on fit results and given limits
+
+    :param fit_summary: Dictionary of the fit output from Multi-Gaussian fit
+    :param peak_lim: Limits on noise peak position
+    :param d0_lim: Limits on distance between noise and first photon peak
+    :param chi2_lim: Limits on reduced chi^2 value
+    :param peak_width_lim: Limits on noise peak width
+    :return: Bad pixel mask
+    """
+    if not fit_summary['is_valid']:
+        return BadPixelsFF.FIT_FAILED.value
+
+    m0 = fit_summary['g0mean']
+    s0 = fit_summary['g0sigma']
+    s1 = fit_summary['g1sigma']
+    s2 = fit_summary['g2sigma']
+    chi2_ndof = fit_summary['chi2_ndof']
+    d01 = fit_summary['g1mean'] - m0
+
+    mask = 0
+    if not fit_summary['is_valid']:
+        mask |= BadPixelsFF.FIT_FAILED.value
+
+    if not fit_summary['has_accurate_covar']:
+        mask |= BadPixelsFF.ACCURATE_COVAR.value
+
+    if not peak_lim[0] <= m0 <= peak_lim[1]:
+        mask |= BadPixelsFF.NOISE_PEAK_THRESHOLD.value
+
+    if not d0_lim[0] <= d01 <= d0_lim[1]:
+        mask |= BadPixelsFF.GAIN_THRESHOLD.value
+
+    if not chi2_lim[0] <= chi2_ndof <= chi2_lim[1]:
+        mask |= BadPixelsFF.CHI2_THRESHOLD.value
+
+    width_lim = peak_width_lim[0] * s0
+    inside_s1 = width_lim[0] <= s1 <= width_lim[1]
+
+    width_lim = peak_width_lim[1] * s0
+    inside_s2 = width_lim[0] <= s2 <= width_lim[1]
+
+    if not inside_s1 and inside_s2:
+        mask |= BadPixelsFF.PEAK_WIDTH_THRESHOLD.value
+
+    return mask

cal_tools/cal_tools/enums.py

@@ -28,6 +28,21 @@ class BadPixels(Enum):
     NON_LIN_RESPONSE_REGION  = 0b100000000000000000000 # bit 21
     
     
+class BadPixelsFF(Enum):
+    """ The SLopesFF Bad Pixel Encoding
+    """
+        
+    FIT_FAILED               = 0b000000000000000000001 # bit 1
+    CHI2_THRESHOLD           = 0b000000000000000000010 # bit 2
+    NOISE_PEAK_THRESHOLD     = 0b000000000000000000100 # bit 3
+    GAIN_THRESHOLD           = 0b000000000000000001000 # bit 4
+    PEAK_WIDTH_THRESHOLD     = 0b000000000000000010000 # bit 5
+    ACCURATE_COVAR           = 0b000000000000000100000 # bit 6
+    BAD_DARK                 = 0b000000000000001000000 # bit 6
+    NO_ENTRY                 = 0b000000000000010000000 # bit 7
+    GAIN_DEVIATION           = 0b000000000000100000000 # bit 8
+    
+    
 class SnowResolution(Enum):
     """ An Enum specifying how to resolve snowy pixels
     """

notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb

@@ -38,6 +38,8 @@
     "karabo_id_control = \"HED_EXP_AGIPD500K2G\" # karabo-id for control device\n",
     "karabo_da_control = 'AGIPD500K2G00' # karabo DA for control infromation\n",
     "\n",
+    "slopes_ff_from_files = \"\" # Path to locally stored SlopesFF and BadPixelsFF constants\n",
+    "\n",
     "use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
     "cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
     "cal_db_timeout = 30000 # in milli seconds\n",

notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb

@@ -40,6 +40,7 @@
     "cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
     "creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
     "\n",
+    "slopes_ff_from_files = \"\" # Path to locally stored SlopesFF and BadPixelsFF constants\n",
     "calfile =  \"\" # path to calibration file. Leave empty if all data should come from DB\n",
     "nodb = False # if set only file-based constants will be used\n",
     "mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
@@ -93,17 +94,13 @@
     "import sys\n",
     "from collections import OrderedDict\n",
     "\n",
-    "# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
     "import os\n",
     "import h5py\n",
     "import numpy as np\n",
     "import matplotlib\n",
     "matplotlib.use(\"agg\")\n",
     "import matplotlib.pyplot as plt\n",
-    "from ipyparallel import Client\n",
-    "print(f\"Connecting to profile {cluster_profile}\")\n",
-    "view = Client(profile=cluster_profile)[:]\n",
-    "view.use_dill()\n",
+    "import multiprocessing as mp\n",
     "\n",
     "from iCalibrationDB import Constants, Conditions, Detectors\n",
     "from cal_tools.tools import (map_modules_from_folder, get_dir_creation_date)\n",
@@ -236,6 +233,7 @@
     "from functools import partial\n",
     "import yaml\n",
     "\n",
+    "\n",
     "def retrieve_constants(karabo_id, bias_voltage, max_cells, acq_rate, \n",
     "                       gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
     "                       cal_db_interface, creation_time, \n",
@@ -294,7 +292,7 @@
     "        acq_rate = get_acq_rate((f, karabo_id, idx))\n",
     "\n",
     "    # avoid creating retireving constant, if requested.\n",
-    "    if not nodb_with_dark:  \n",
+    "    if not nodb_with_dark:\n",
     "        const_dict = assemble_constant_dict(corr_bools, pc_bools, max_cells, bias_voltage,\n",
     "                                            gain_setting, acq_rate, photon_energy,\n",
     "                                            beam_energy=None, only_dark=only_dark)\n",
@@ -303,26 +301,32 @@
     "        # to return a dict of useful metadata.\n",
     "        mdata_dict = dict()\n",
     "        for cname, cval in const_dict.items():\n",
-    "            try:\n",
-    "                condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
-    "                co, mdata = \\\n",
-    "                    get_from_db(dev, getattr(Constants.AGIPD, cname)(),\n",
-    "                                condition, getattr(np, cval[0])(cval[1]),\n",
-    "                                cal_db_interface, creation_time, meta_only=True, verbosity=0)\n",
-    "                mdata_const = mdata.calibration_constant_version\n",
-    "                # saving metadata in a dict\n",
-    "                mdata_dict[cname] = dict()\n",
-    "                # check if constant was sucessfully retrieved.\n",
-    "                if mdata.comm_db_success:                       \n",
-    "                    mdata_dict[cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
-    "                                                     f\"{mdata_const.filename}\"\n",
-    "                    mdata_dict[cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
-    "                else:\n",
-    "                    mdata_dict[cname][\"file-path\"] = const_dict[cname][:2]\n",
-    "                    mdata_dict[cname][\"creation-time\"] = None\n",
-    "            except Exception as e:\n",
-    "                err = f\"Error: {e}, Traceback: {traceback.format_exc()}\"\n",
-    "                print(err)\n",
+    "            # saving metadata in a dict\n",
+    "            mdata_dict[cname] = dict()\n",
+    "            \n",
+    "            if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
+    "                mdata_dict[cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
+    "                mdata_dict[cname][\"creation-time\"] = \"00:00:00\"\n",
+    "            else:\n",
+    "                try:\n",
+    "                    condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
+    "                    co, mdata = \\\n",
+    "                        get_from_db(dev, getattr(Constants.AGIPD, cname)(),\n",
+    "                                    condition, getattr(np, cval[0])(cval[1]),\n",
+    "                                    cal_db_interface, creation_time, meta_only=True, verbosity=0)\n",
+    "                    mdata_const = mdata.calibration_constant_version\n",
+    "                    \n",
+    "                    # check if constant was sucessfully retrieved.\n",
+    "                    if mdata.comm_db_success:                       \n",
+    "                        mdata_dict[cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
+    "                                                         f\"{mdata_const.filename}\"\n",
+    "                        mdata_dict[cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+    "                    else:\n",
+    "                        mdata_dict[cname][\"file-path\"] = const_dict[cname][:2]\n",
+    "                        mdata_dict[cname][\"creation-time\"] = None\n",
+    "                except Exception as e:\n",
+    "                    err = f\"Error: {e}, Traceback: {traceback.format_exc()}\"\n",
+    "                    print(err)\n",
     "\n",
     "    return qm, mdata_dict, dev.device_name, acq_rate, max_cells, err\n",
     "\n",
@@ -360,8 +364,9 @@
     "            cal_db_interface, creation_time, \n",
     "            corr_bools, pc_bools)\n",
     "\n",
-    "results = view.map_sync(p, inp)\n",
-    "#results = list(map(p, inp))\n",
+    "with mp.Pool(processes=16) as pool:\n",
+    "    results = pool.map(p, inp)\n",
+    "\n",
     "mod_dev = dict()\n",
     "mdata_dict = dict()\n",
     "for r in results:\n",
@@ -433,13 +438,6 @@
     "with open(f\"{out_folder}/retrieved_constants.yml\",\"w\") as fyml:\n",
     "        yaml.safe_dump(time_summary, fyml)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

tests/test_agipdutils_ff.py

+import numpy as np
+from cal_tools.agipdutils_ff import get_mask, set_par_limits
+
+
+def test_get_mask():
+    fit_summary = {
+        'chi2_ndof': 1.674524751845516,
+        'g0n': 6031.641198873036,
+        'error_g0n': 94.63055028459667,
+        'limit_g0n': np.array([0.0, None]),
+        'fix_g0n': False,
+        'g0mean': -13.711814669099589,
+        'error_g0mean': 0.2532017427306297,
+        'limit_g0mean': np.array([-30,  30]),
+        'fix_g0mean': False,
+        'g0sigma': 13.478502058651568,
+        'error_g0sigma': 0.2588135637661919,
+        'limit_g0sigma': np.array([0, 30]),
+        'fix_g0sigma': False,
+        'g1n': 4337.126861254491,
+        'error_g1n': 69.764180118274,
+        'limit_g1n': np.array([0, None]),
+        'fix_g1n': False,
+        'g1mean': 53.90265411499657,
+        'error_g1mean': 0.27585684670864746,
+        'limit_g1mean': None,
+        'fix_g1mean': False,
+        'g1sigma': 15.687448834904817,
+        'error_g1sigma': 0.2951166525483524,
+        'limit_g1sigma': np.array([0, 35]),
+        'fix_g1sigma': False,
+        'g2n': 1542.531700635003,
+        'error_g2n': 43.20145030604567,
+        'limit_g2n': np.array([0, None]),
+        'fix_g2n': False,
+        'g2mean': 120.98535387591575,
+        'error_g2mean': 0.509566354942716,
+        'limit_g2mean': None,
+        'fix_g2mean': False,
+        'g2sigma': 15.550452880533143,
+        'error_g2sigma': 0.5003254358001863,
+        'limit_g2sigma': np.array([0, 40]),
+        'fix_g2sigma': False,
+        'g3n': 1261189.2282171287,
+        'error_g3n': 1261190.2282163086,
+        'limit_g3n': np.array([0, None]),
+        'fix_g3n': False,
+        'g3mean': 348.68766379647343,
+        'error_g3mean': 17.23872285713375,
+        'limit_g3mean': None,
+        'fix_g3mean': False,
+        'g3sigma': 44.83987230934497,
+        'error_g3sigma': 30.956164693249242,
+        'limit_g3sigma': np.array([0, 45]),
+        'fix_g3sigma': False,
+        'fval': 336.5794751209487,
+        'edm': 0.00011660826330754263,
+        'tolerance': 0.1,
+        'nfcn': 4620,
+        'ncalls': 4620,
+        'up': 1.0,
+        'is_valid': True,
+        'has_valid_parameters': True,
+        'has_accurate_covar': True,
+        'has_posdef_covar': True,
+        'has_made_posdef_covar': False,
+        'hesse_failed': False,
+        'has_covariance': True,
+        'is_above_max_edm': False,
+        'has_reached_call_limit': False}
+    peak_lim = [-30, 30]
+    d0_lim = [10, 80]
+    chi2_lim = [0, 3.0]
+    peak_width_lim = np.array([[0.9, 1.55], [0.95, 1.65]])
+    mask = get_mask(fit_summary, peak_lim, d0_lim, chi2_lim, peak_width_lim)
+    assert mask == 0
+
+
+def test_set_par_limits():
+    peak_range = np.array([[-30, 30],
+                           [35, 70],
+                           [95, 135],
+                           [145,  220]])
+
+    peak_norm_range = np.array([[0.0, None],
+                                [0, None],
+                                [0, None],
+                                [0, None]])
+    peak_width_range = np.array([[0, 30],
+                                 [0, 35],
+                                 [0, 40],
+                                 [0, 45]])
+
+    parameters = {
+        'g0sigma': 9.620186459204016,
+        'g0n': 5659.0,
+        'g0mean': -3.224686340342817,
+        'g1sigma': 8.149415371586683,
+        'g1n': 3612.0,
+        'g1mean': 54.6281838316722,
+        'g2sigma': 9.830124777667839,
+        'g2n': 1442.0,
+        'g2mean': 114.92510402219139,
+        'g3sigma': 15.336595220228498,
+        'g3n': 474.0,
+        'g3mean': 167.0295358649789}
+
+    expected = {
+        'g0sigma': 9.620186459204016,
+        'g0n': 5659.0,
+        'g0mean': -3.224686340342817,
+        'g1sigma': 8.149415371586683,
+        'g1n': 3612.0,
+        'g1mean': 54.6281838316722,
+        'g2sigma': 9.830124777667839,
+        'g2n': 1442.0,
+        'g2mean': 114.92510402219139,
+        'g3sigma': 15.336595220228498,
+        'g3n': 474.0,
+        'g3mean': 167.0295358649789,
+        'limit_g0n': np.array([0.0, None]),
+        'limit_g0mean': np.array([-30,  30]),
+        'limit_g0sigma': np.array([0, 30]),
+        'limit_g1n': np.array([0, None]),
+        'limit_g1mean': np.array([35, 70]),
+        'limit_g1sigma': np.array([0, 35]),
+        'limit_g2n': np.array([0, None]),
+        'limit_g2mean': np.array([95, 135]),
+        'limit_g2sigma': np.array([0, 40]),
+        'limit_g3n': np.array([0, None]),
+        'limit_g3mean': np.array([145, 220]),
+        'limit_g3sigma': np.array([0, 45])}
+
+    set_par_limits(parameters, peak_range, peak_norm_range, peak_width_range)
+    assert parameters.keys() == expected.keys()
+    for key in parameters.keys():
+        if isinstance(parameters[key], np.ndarray):
+            assert np.all(parameters[key] == expected[key])
+        else:
+            assert parameters[key] == expected[key]

xfel_calibrate/notebooks.py

@@ -21,6 +21,8 @@ notebooks = {
         "FF": {
             "notebook":
                 "notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb",
+            "dep_notebooks": [
+                "notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb"],
             "concurrency": {"parameter": "modules",
                             "default concurrency": 16,
                             "cluster cores": 16},