Agipd corr mod

Summary:

These are changes introduced by @jsztuk for AGIPD correction

• Relative gain correction modification.
• exposing common-mode configurations.
• updates for the within code documentation.

The MR was rebased with the master and I have refactored @jsztuk 's chages and it is ready for review

@danilevc @kamile

cal_tools/cal_tools/agipdlib.py

@@ -273,7 +273,7 @@ class AgipdCorrections:
             f = h5py.File(file_name, 'r')
             group = f[data_path]
 
-            valid, first_index, last_index, valid_trains, valid_indices = \
+            _, first_index, last_index, __, valid_indices = \
                 self.get_valid_image_idx(idx_base, f)
             firange = self.gen_valid_range(first_index, last_index,
                                            self.max_cells, agipd_base, f,
@@ -344,7 +344,8 @@ class AgipdCorrections:
                 arr = data_dict[field][:n_img]
                 kw = {'fletcher32': True}
                 if field in compress_fields:
-                    kw.update(compression='gzip', compression_opts=1, shuffle=True)
+                    kw.update(compression='gzip', compression_opts=1,
+                              shuffle=True)
                 if arr.ndim > 1:
                     kw['chunks'] = (1,) + arr.shape[1:]  # 1 chunk = 1 image
 
@@ -376,8 +377,8 @@ class AgipdCorrections:
         n_img = self.shared_dict[i_proc]['nImg'][0]
         cell_ids = cell_id[train_id == train_id[0]]
         n_cells = cell_ids.size
-        data = self.shared_dict[i_proc]['data'][:n_img].reshape(-1, n_cells, 8,
-                                                                64, 2, 64)
+        data = self.shared_dict[i_proc]['data'][:n_img].reshape(-1, n_cells,
+                                                                8, 64, 2, 64)
 
         # Loop over iterations
         for i in range(n_itr):
@@ -391,7 +392,8 @@ class AgipdCorrections:
 
                 # Cell common mode
                 cell_cm_sum, cell_cm_count = \
-                    calgs.sum_and_count_in_range_cell(asic_data, dark_min, dark_max)
+                    calgs.sum_and_count_in_range_cell(asic_data, dark_min,
+                                                      dark_max)
                 cell_cm = cell_cm_sum / cell_cm_count
 
                 cell_cm[cell_cm_count < fraction * 32 * 256] = 0
@@ -399,7 +401,8 @@ class AgipdCorrections:
 
                 # Asics common mode
                 asic_cm_sum, asic_cm_count = \
-                    calgs.sum_and_count_in_range_asic(asic_data, dark_min, dark_max)
+                    calgs.sum_and_count_in_range_asic(asic_data, dark_min,
+                                                      dark_max)
                 asic_cm = asic_cm_sum / asic_cm_count
 
                 asic_cm[asic_cm_count < fraction * 64 * 64] = 0
@@ -565,7 +568,8 @@ class AgipdCorrections:
             del mgbc
 
         # Do xray correction if requested 
-        # JSD: The slopes we have in our constants are already relative slopeFF = slopeFFpix/avarege(slopeFFpix)
+        # The slopes we have in our constants are already relative
+        # slopeFF = slopeFFpix/avarege(slopeFFpix)
         # To apply them we have to / not *
         if self.corr_bools.get("xray_corr"):
             data /= self.xray_cor[module_idx]
@@ -757,8 +761,8 @@ class AgipdCorrections:
             uq, fidxv, cntsv = np.unique(trains, return_index=True,
                                          return_counts=True)
 
-            # Validate calculated CORR INDEX contents by checking difference between
-            # trainId stored in RAW data and trains from
+            # Validate calculated CORR INDEX contents by checking 
+            # difference between trainId stored in RAW data and trains from
             train_diff = np.isin(np.array(infile["/INDEX/trainId"]), uq,
                                  invert=True)
 
@@ -844,27 +848,29 @@ class AgipdCorrections:
         exists of the current AGIPD instances.
 
         Relative gain is derived both from pulse capacitor as well as low
-        intensity  flat field data, information from ff data is needed to 
-        'calibrate' pulse capacitor data, if there is no available FF data,
-        relative gain for High Gain stage is set to 1:
+        intensity flat field data, information from flat field data is 
+        needed to 'calibrate' pulse capacitor data, if there is no 
+        available FF data, relative gain for High Gain stage is set to 1:
 
-        * Relative gain for High gain stage - from  the FF data we get the relative slopes 
-          of a given pixel and memory cells with respect to all memory cells and all pixels in the module,
-          Please note: Current slopesFF avaialble in calib. constants are created per pixel only, 
-          not per memory cell:
+        * Relative gain for High gain stage - from the FF data we get 
+          the relative slopes of a given pixel and memory cells with 
+          respect to all memory cells and all pixels in the module,
+          Please note: Current slopesFF avaialble in calibibration
+          constants are created per pixel only, not per memory cell:
 
              rel_high_gain = 1 if only PC data is available
              rel_high_gain = rel_slopesFF if FF data is also available
 
           
 
-        * Relaitive gain for Medium gain stage: we derive the factor between high and medium gain
-          using slope information from fits to the linear part of high and medium gain:
+        * Relative gain for Medium gain stage: we derive the factor
+          between high and medium gain using slope information from
+          fits to the linear part of high and medium gain:
 
              rfpc_high_medium = m_h/m_m          
 
-          where m_h and m_m is the medium gain slope of given memory cells and 
-          pixel and m_h is the high gain slope as above
+          where m_h and m_m is the medium gain slope of given memory cells 
+          and pixel and m_h is the high gain slope as above
            rel_gain_medium = rel_high_gain * rfpc_high_medium
 
         With this data the relative gain for the three gain stages evaluates
@@ -909,11 +915,16 @@ class AgipdCorrections:
             else:
                 xray_cor = np.squeeze(slopesFF[..., 0])
 
-            # relative X-ray correction is normalized by the median of all pixels
-            # JSD: to be checked: why it is again divided by median. If we have relative slopes in the constants
-            # and (we have!) xray cor = (slopeFF/avarege_slopeFF)/avarege_slopeFF. It does not make any sense
-            # to me. I remove it. 
-#            xray_cor /= np.nanmedian(xray_cor)
+            # relative X-ray correction is normalized by the median
+            # of all pixels
+
+            # TODO: A check is required to know why it is again divided by 
+            # median. If we have relative slopes in the constants
+            # and (we have!) 
+            # xray cor = (slopeFF/avarege_slopeFF)/avarege_slopeFF.
+            # It didn't not make sense and was removed. 
+            # xray_cor /= np.nanmedian(xray_cor)
+
             self.xray_cor[module_idx][...] = xray_cor.transpose()[...]
 
         # add additional bad pixel information
@@ -923,7 +934,7 @@ class AgipdCorrections:
 
             slopesPC = cons_data["SlopesPC"].astype(np.float32)
 
-            # this will handle some historical data in a different format
+            # This will handle some historical data in a different format
             # constant dimension injected first
             if slopesPC.shape[0] == 10 or slopesPC.shape[0] == 11:
                 slopesPC = np.moveaxis(slopesPC, 0, 3)
@@ -941,23 +952,28 @@ class AgipdCorrections:
             # calculate ratio high to medium gain 
             pc_high_ave = np.nanmean(pc_high_m, axis=(0,1))
             pc_med_ave = np.nanmean(pc_med_m, axis=(0,1))
-            # ration between HG and MG per pixel per mem cell used for rel gain calculation
+            # ration between HG and MG per pixel per mem cell 
+            # used for rel gain calculation
             frac_high_med_pix = pc_high_m / pc_med_m
-            # avarage ration between HG and MG as a function of mem cell (needed for bls_stripes)
-            # I do not think it is a optimal correction, why it is not per pixel
+            # avarage ration between HG and MG as a function of 
+            # mem cell (needed for bls_stripes)
+            # TODO: Per pixel would be more optimal correction
             frac_high_med = pc_high_ave / pc_med_ave
             # calculate additional medium-gain offset
             md_additional_offset = pc_high_l - pc_med_l * pc_high_m / pc_med_m
 
-            # JSD:  Calculate relative gain. If FF constants are available, use them for high gain
+            # Calculate relative gain. If FF constants are available,
+            # use them for high gain
             # if not rel_gain is calculated using PC data only
-#            if self.corr_bools.get("xray_corr"):
-#                rel_gain[..., :self.max_cells, 0] /= xray_corr 
+            # if self.corr_bools.get("xray_corr"):
+            #     rel_gain[..., :self.max_cells, 0] /= xray_corr 
+
+            # PC data should be 'calibrated with X-ray data, 
+            # if it is not done, it is better to use 1 instead of bias 
+            # the results with PC arteffacts. 
+            # rel_gain[..., 0] = 1./(pc_high_m / pc_high_ave)
 
-#         PC data should be 'calibrated with X-ray data, if it is not done, it is better to use 1 instead of bias the 
-#         the results with PC arteffacts. 
-#            rel_gain[..., 0] = 1./(pc_high_m / pc_high_ave)
-            rel_gain[..., 0] = rel_gain[..., 0]
+            # High-gain (rel_gain[..., 0]) stays the same
             rel_gain[..., 1] = rel_gain[..., 0] * frac_high_med_pix
             rel_gain[..., 2] = rel_gain[..., 1] * 4.48