fixing agipd corr and agipd pc

MR ready for review @karnem

1. fixes for AGIPD CORR
2. concurrency parameter
3. JungFrau correction for old raw data 2018
4. installation
5. extra message for having same previous constant.

cal_tools/cal_tools/agipdlib.py

 import copy
 from enum import Enum
 import os
-
+import gc
 import h5py
 import numpy as np
 from scipy.signal import cwt, ricker
@@ -13,6 +13,8 @@ from cal_tools.enums import BadPixels
 from cal_tools.tools import get_constant_from_db, get_constant_from_db_and_time
 from iCalibrationDB import Constants, Conditions, Detectors
 
+import logging
+
 
 def get_num_cells(fname, loc, module):
     with h5py.File(fname, "r") as f:
@@ -114,7 +116,7 @@ class AgipdCorrections:
                  cal_det_instance="AGIPD1M1", karabo_data_mode=False,
                  force_hg_if_below=None, force_mg_if_below=None,
                  mask_noisy_adc=False, acquisition_rate=None, gain_setting=None,
-                 corr_bools=None):
+                 corr_bools=None, logfile=None):
         """
         Initialize an AgipdCorrections Class
 
@@ -192,6 +194,28 @@ class AgipdCorrections:
         self.frac_high_med = 0
         self.md_additional_offset = 0
         self.xray_cor = 0
+        self.logfile = logfile
+
+        self.logger = logging.getLogger('server_logger2')
+        self.logger.setLevel(logging.DEBUG)
+        # create file handler which logs even debug messages
+        fh = logging.FileHandler(logfile)
+        fh.setLevel(logging.DEBUG)
+        # create console handler with a higher log level
+        ch = logging.StreamHandler()
+        ch.setLevel(logging.ERROR)
+        # create formatter and add it to the handlers
+        formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
+        ch.setFormatter(formatter)
+        fh.setFormatter(formatter)
+        # add the handlers to logger
+        self.logger.addHandler(ch)
+        self.logger.addHandler(fh)
+        fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        logging.basicConfig(filename=logfile,
+                        level=getattr(logging, "DEBUG"),
+                        format=fmt)
+
 
         # check if given corr_bools are correct
         tot_corr_bools = ['only_offset', 'adjust_mg_baseline', 'rel_gain',
@@ -323,12 +347,6 @@ class AgipdCorrections:
             print("Threshold medians are {}".format(
                 np.nanmedian(self.thresholds, axis=(0, 1, 2))))
 
-        # delete unneeded parameters.
-        todel = [offset, rel_gain, xray_cor, mask, noise, thresholds,
-                 swap_axis, frac_high_med, md_additional_offset]
-        for d in todel:
-            if d is not None:
-                del d
     def get_shadowed_stripe(self, data, threshold, fraction):
         """
         Return list of shadowed regions.
@@ -914,7 +932,7 @@ class AgipdCorrections:
         """
         if not self.initialized:
             raise RuntimeError("Must call initialize() first!")
-
+        self.logger.debug("check init \n")
         if not self.karabo_data_mode:
             agipd_base = self.agipd_base
             cidx = self.cidx
@@ -935,6 +953,7 @@ class AgipdCorrections:
             # this far end of the image index range we are working on
             nidx = int(cidx + irange.size)
         else:
+            self.logger.debug("Not karabo mode \n")
             cidx = 1  # do not produce any histograms
             im = irange['image.data']
             trainId = np.squeeze(irange['image.trainId'])
@@ -943,12 +962,15 @@ class AgipdCorrections:
             cellid = np.squeeze(irange['image.cellId'])
             length = np.squeeze(irange['image.length'])
 
+        self.logger.debug("splitting image \n")
         # split off image and gain into two separate arrays
         im, ga = self.gsfun(im)
 
+        self.logger.debug("float image \n")
         # we will work on float data from now on
         im = im.astype(np.float32)
 
+        self.logger.debug("copy raw \n")
         # some correction require us to maintain a copy of the raw data
         raw = copy.copy(im)
 
@@ -961,6 +983,7 @@ class AgipdCorrections:
             length = length[0::2]
 
         # first evaluate the gain into 0, 1, 2 --> high, medium, low
+        self.logger.debug("evaluating gain \n")
         gain = np.zeros(ga.shape, np.uint8)
         gain[...] = 0
         t0 = self.thresholds[..., 0]
@@ -971,11 +994,12 @@ class AgipdCorrections:
         gain[(ga > t1[cellid, ...]) & (
                 t1[cellid, ...] > 1.05 * t0[cellid, ...])] = 2
 
+        self.logger.debug("check  force_mg_if_below \n")
         # force into high or medium gain if requested
         if self.force_mg_if_below is not None and self.force_mg_if_below > 0:
             gain[(gain == 2) & ((im - self.offset[
                 cellid, ..., 1]) < self.force_mg_if_below)] = 1
-
+        self.logger.debug("check  force_hg_if_below \n")
         if self.force_hg_if_below is not None and self.force_hg_if_below > 0:
             gain[(gain > 0) & ((im - self.offset[
                 cellid, ..., 0]) < self.force_hg_if_below)] = 0
@@ -986,6 +1010,7 @@ class AgipdCorrections:
         # check if any data has zero standard deviation, and mark this in
         # the bad pixel maks
         # this can be done on otherwise not corrected data.
+        self.logger.debug("check  if any data has zero standard deviation \n")
         if self.sig_zero_mask is None:
             self.sig_zero_mask = np.zeros(
                 (self.max_cells, im.shape[1], im.shape[2]), np.uint32)
@@ -993,7 +1018,7 @@ class AgipdCorrections:
                 std = np.nanstd(im[cellid == c, ...], axis=0)
                 self.sig_zero_mask[
                     c, std == 0] = BadPixels.DATA_STD_IS_ZERO.value
-
+        self.logger.debug("check the first chunk \n")
         # for feedback we produced histograms for the first chunk
         if cidx == 0:
             H, xe, ye = np.histogram2d(im.flatten(), ga.flatten(),
@@ -1007,31 +1032,31 @@ class AgipdCorrections:
                                        range=[[4000, 8192], [0, 4]])
             self.hists_dig_gain_vs_signal += H
             self.dig_signal_edges = (xe, ye)
-
+        self.logger.debug("check correct constants depending \n")
         # now get the correct constants depending on cell id
         offsetb = self.offset[cellid, ...]
         tmask = self.mask[cellid, ...]
-
+        self.logger.debug("choose constants according to gain setting \n")
         # choose constants according to gain setting
         off = np.choose(gain,
                         (offsetb[..., 0], offsetb[..., 1], offsetb[..., 2]))
 
         msk = np.choose(gain, (tmask[..., 0], tmask[..., 1], tmask[..., 2]))
-
+        self.logger.debug("get the correct rel_gain depending on cell-id \n")
         # same for relative gain and then bad pixel mask
         if hasattr(self, "rel_gain"):
             # get the correct rel_gain depending on cell-id
             rc = self.rel_gain[cellid, ...]
             rel_cor = np.choose(gain, (rc[..., 0], rc[..., 1], rc[..., 2]))
-
+        self.logger.debug("scale raw gain for use in the identifying snowy pixels \n")
         # scale raw gain for use in the identifying snowy pixels
         rgain = None
         if self.melt_snow is not False:
             rgain = ga / t0[cellid, ...]
-
+        self.logger.debug("subtract offset \n")
         # subtract offset
         im -= off
-
+        self.logger.debug("evaluate any baseline shifts \n")
         # before doing relative gain correction we need to evaluate any
         # baseline shifts
         # as they are effectively and additional offset in the data
@@ -1083,22 +1108,27 @@ class AgipdCorrections:
 
         # now we can correct for relative gain if requested
         if self.corr_bools.get("rel_gain") and hasattr(self, "rel_gain"):
+            self.logger.debug("correct for relative gain \n")
             im *= rel_cor
 
         if self.corr_bools.get("adjust_mg_baseline"):
+            self.logger.debug("correct adjust_mg_baseline \n")
             mgbc = self.md_additional_offset[cellid, ...]
             im[gain == 1] += mgbc[gain == 1]
 
         # Set negative values for medium gain to 0
         if self.corr_bools.get('blc_set_min'):
+            self.logger.debug("correct blc_set_min \n")
             im[(im < 0) & (gain == 1)] = 0
 
         # Do xray correction if requested
         if self.corr_bools.get("xray_corr"):
+            self.logger.debug("correct xray_corr \n")
             im *= self.xray_cor
 
         # try to identify snowy pixels at this point
         if self.melt_snow is not False:
+            self.logger.debug("correct melt_snow\n")
             ms = self.melt_snow
             im, gain, snowmask = self.melt_snowy_pixels(raw,
                                                         im,
@@ -1129,7 +1159,7 @@ class AgipdCorrections:
         msk |= self.sig_zero_mask[cellid, ...]
         if self.melt_snow is not False:
             msk |= snowmask
-
+        self.logger.debug("for the first chunk output some statistics\n")
         # for the first chunk output some statistics
         if cidx == 0:
             copim = copy.copy(im)
@@ -1164,6 +1194,7 @@ class AgipdCorrections:
 
         # apply noisy ADC mask if requested
         if self.mask_noisy_adc is not None and self.mask_noisy_adc != 0:
+            self.logger.debug("mask_noisy_adc\n")
             if self.adc_mask is None:
                 self.adc_mask = self.make_noisy_adc_mask(msk)
             msk |= self.adc_mask
@@ -1172,16 +1203,26 @@ class AgipdCorrections:
         sd = 1 if not self.il_mode else 2
 
         if not self.karabo_data_mode:
+            self.logger.debug("now write out the data\n")
+            
             self.ddset[cidx // sd:nidx // sd, ...] = im
+            self.logger.debug("1\n")
             self.gdset[cidx // sd:nidx // sd, ...] = gain
+            self.logger.debug("2\n")
             self.mdset[cidx // sd:nidx // sd, ...] = msk
-
+            self.logger.debug("3\n")
             self.outfile[agipd_base + "image/cellId"][cidx:nidx] = cellid
+            self.logger.debug("4\n")
             self.outfile[agipd_base + "image/trainId"][cidx:nidx] = trainId
+            self.logger.debug("5\n")
             self.outfile[agipd_base + "image/pulseId"][cidx:nidx] = pulseId
+            self.logger.debug("6\n")
             self.outfile[agipd_base + "image/status"][cidx:nidx] = status
+            self.logger.debug("7\n")
             self.outfile[agipd_base + "image/length"][cidx:nidx] = length
+            self.logger.debug("8\n")
             self.cidx = nidx
+            self.logger.debug("now write out the data-finished\n")
         else:
             irange['image.data'] = im
             irange['image.gain'] = gain
@@ -1193,6 +1234,7 @@ class AgipdCorrections:
             irange['image.length'] = length
             return irange
 
+
     def get_valid_image_idx(self):
         """ Return the indices of valid data
         """
@@ -1627,7 +1669,7 @@ class AgipdCorrections:
             # Calculate relative gain
             rel_gain[..., 0] = pc_high_m / pc_high_ave
             rel_gain[..., 1] = pc_med_m / pc_med_ave * frac_high_med
-            rel_gain[..., 2] = rel_gain[..., 1] * 0.233
+            rel_gain[..., 2] = rel_gain[..., 1] * 4.48
         else:
             md_additional_offset = None
             rel_gain = None