From 86e1009d63b012c84838f89e977947ca2fae4b36 Mon Sep 17 00:00:00 2001
From: Steffen Hauf <steffen.hauf@xfel.eu>
Date: Thu, 5 Sep 2019 15:12:27 +0200
Subject: [PATCH 1/5] Mark non-linear regions in BP mask

---
 cal_tools/cal_tools/lpdlib.py              | 36 +++++++++++++++++++---
 cal_tools/cal_tools/tools.py               |  1 +
 notebooks/LPD/LPD_Correct_and_Verify.ipynb | 23 ++++++++++----
 3 files changed, 49 insertions(+), 11 deletions(-)

diff --git a/cal_tools/cal_tools/lpdlib.py b/cal_tools/cal_tools/lpdlib.py
index d0c557aab..fff0d401c 100644
--- a/cal_tools/cal_tools/lpdlib.py
+++ b/cal_tools/cal_tools/lpdlib.py
@@ -41,7 +41,8 @@ class LpdCorrections:
                  raw_fmt_version=2, chunk_size=512,
                  h5_data_path="INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/",
                  h5_index_path="INDEX/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/",
-                 do_ff=True, correct_non_linear=True, karabo_data_mode=False):
+                 do_ff=True, correct_non_linear=True, karabo_data_mode=False,
+                 linear_between=None, mark_non_lin_region=True, nlc_version=2):
         """
         Initialize an LpdCorrections Class
 
@@ -89,6 +90,9 @@ class LpdCorrections:
         self.filter_cells = [0, 1] if channel in filter_modules else []
         self.cnl = True  # correct_non_linear
         self.karabo_data_mode = karabo_data_mode
+        self.linear_between = linear_between
+        self.mark_nonlin = mark_non_lin_region
+        self.nlc_version = nlc_version
         # emprically determined from APD datasets p900038, r155,r156
         # emprically determined from APD datasets p900038, r155,r156
         self.cnl_const = {
@@ -252,9 +256,24 @@ class LpdCorrections:
 
         # correct offset
         im -= og
+        
+        nlf = 0
+        if self.mark_nonlin:
+            for gl, lr in enumerate(self.linear_between):
+                
+                midx = (gain == gl) & ((im < lr[0]) | (im > lr[1]))
+                msk[midx] = BadPixels.NON_LIN_RESPONSE_REGION.value
+                numnonlin = np.count_nonzero(midx, axis=(1,2))
+                nlf += numnonlin
+            print(nlf.shape)
+            nlf = nlf/float(im.shape[0] * im.shape[1])
+            print("Marked non-linear-regions")
 
         # hacky way of smoothening transition region between med and low
-        cfac = 0.314 * np.exp(-im * 0.001)
+        
+        cfac = 1
+        if self.nlc_version == 1 and self.cnl:
+            cfac = 0.314 * np.exp(-im * 0.001)
 
         # perform relative gain correction with additional gain-deduced
         # offset
@@ -263,10 +282,11 @@ class LpdCorrections:
             im /= self.flatfield[None, :, :]
 
         # hacky way of smoothening transition region between med and low
-        # im[gain == 2] -= im[gain == 2] * cfac[gain == 2]
+        if self.nlc_version == 1 and self.cnl:
+            im[gain == 2] -= im[gain == 2] * cfac[gain == 2]
 
         # perform non-linear corrections if requested
-        if self.cnl:
+        if self.cnl and self.nlc_version == 2:
             def lin_exp_fun(x, m, b, A, lam, c):
                 return m * x + b + A * np.exp(lam * (x - c))
 
@@ -287,7 +307,7 @@ class LpdCorrections:
             cf = lin_exp_fun(x, cnl['m'], cnl['b'], cnl['A'], cnl['lam'],
                              cnl['c'])
             im[(gain == 2)] -= np.minimum(cf, 0.45) * x
-
+        
         # create bad pixels masks, here non-finite values
         bidx = ~np.isfinite(im)
         im[bidx] = 0
@@ -332,6 +352,8 @@ class LpdCorrections:
             self.outfile[lpd_base + "image/pulseId"][cidx:nidx] = pulseId
             self.outfile[lpd_base + "image/status"][cidx:nidx] = status
             self.outfile[lpd_base + "image/length"][cidx:nidx] = length
+            if self.mark_nonlin:
+                self.outfile[lpd_base + "image/nonLinear"][cidx:nidx] = nlf
             self.cidx = nidx
         else:
             irange['image.data'] = im
@@ -522,6 +544,10 @@ class LpdCorrections:
                                     dtype=np.uint16, fletcher32=True)
         self.outfile.create_dataset(lpdbase + "image/length", fsz,
                                     dtype=np.uint32, fletcher32=True)
+        
+        if self.mark_nonlin:
+            self.outfile.create_dataset(lpdbase + "image/nonLinear", fsz,
+                                        dtype=np.float32, fletcher32=True)
 
     def get_histograms(self):
         """ Return preview histograms computed from the first chunk
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index 8036b7ac4..3ca11ac74 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -487,6 +487,7 @@ def get_from_db(device, constant, condition, empty_constant,
     from iCalibrationDB import ConstantMetaData, Versions
     import zmq
     
+    timeout=1200000
     if version_info:
         meta_only = False
     
diff --git a/notebooks/LPD/LPD_Correct_and_Verify.ipynb b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
index 8cb6da495..acee0ff45 100644
--- a/notebooks/LPD/LPD_Correct_and_Verify.ipynb
+++ b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
@@ -20,9 +20,9 @@
    },
    "outputs": [],
    "source": [
-    "in_folder = \"/gpfs/exfel/exp/FXE/201931/p900088/raw/\" # the folder to read data from, required\n",
+    "in_folder = \"/gpfs/exfel/exp/FXE/201802/p002218/raw/\" # the folder to read data from, required\n",
     "run = 115 # runs to process, required\n",
-    "out_folder = \"/gpfs/exfel/exp/FXE/201931/p900088/proc/\" # the folder to output to, required\n",
+    "out_folder = \"/gpfs/exfel/data/scratch/xcal/lpd_test/exclude\" # the folder to output to, required\n",
     "calfile =  \"/gpfs/exfel/data/scratch/xcal/lpd_store_0519.h5\" # path to constants extracted from the db into a file\n",
     "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
     "mem_cells = 512 # memory cells in data\n",
@@ -47,6 +47,11 @@
     "beam_center_offset =  [1.5, 1] # offset from the beam center, MAR 2018\n",
     "sequences_per_node = 1 # sequence files to process per node\n",
     "timeout_cal_db = 30000 # timeout for calibration db requests in milliseconds\n",
+    "dont_mark_non_lin_region = False # do not mark non-linear regions in BP map\n",
+    "linear_between_high_gain = [-5000, 2500]  # region in which high gain is considered linear\n",
+    "linear_between_med_gain = [300, 3000]  # region in which medium gain is considered linear\n",
+    "linear_between_low_gain = [300, 3000]  # region in which low gain is considered linear\n",
+    "nlc_version = 2 # version of NLC to use\n",
     "\n",
     "def balance_sequences(in_folder, run, sequences, sequences_per_node):\n",
     "    import glob\n",
@@ -155,7 +160,10 @@
     "logger = InfluxLogger(detector=\"LPD\", instrument=instrument, mem_cells=mem_cells,\n",
     "                      notebook=get_notebook_name(), proposal=proposal)\n",
     "\n",
-    "client = InfluxDBClient('exflqr18318', 8086, 'root', 'root', 'calstats')"
+    "client = InfluxDBClient('exflqr18318', 8086, 'root', 'root', 'calstats')\n",
+    "\n",
+    "mark_non_lin_region = not dont_mark_non_lin_region\n",
+    "linear_between = [linear_between_high_gain, linear_between_med_gain, linear_between_low_gain]"
    ]
   },
   {
@@ -271,7 +279,8 @@
     "from functools import partial\n",
     "def correct_module(max_cells, do_ff, index_v, CHUNK_SIZE, total_sequences, sequences_qm, \n",
     "                   bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range, max_pulses,\n",
-    "                   dbparms, fileparms, nodb, no_non_linear_corrections, inp):\n",
+    "                   dbparms, fileparms, nodb, no_non_linear_corrections, mark_non_lin_region, linear_between,\n",
+    "                   nlc_version, inp):\n",
     "    import numpy as np\n",
     "    import copy\n",
     "    import h5py\n",
@@ -333,7 +342,9 @@
     "        lpd_corr = LpdCorrections(infile, outfile, max_cells, channel, max_pulses,\n",
     "                                  bins_gain_vs_signal, bins_signal_low_range,\n",
     "                                  bins_signal_high_range, do_ff=do_ff, raw_fmt_version=index_v,\n",
-    "                                  correct_non_linear=(not no_non_linear_corrections))\n",
+    "                                  correct_non_linear=(not no_non_linear_corrections),\n",
+    "                                  mark_non_lin_region=mark_non_lin_region, linear_between=linear_between,\n",
+    "                                  nlc_version=nlc_version)\n",
     "        \n",
     "        try:\n",
     "            lpd_corr.get_valid_image_idx()            \n",
@@ -410,7 +421,7 @@
     "        print(\"Running {} tasks parallel\".format(len(inp)))\n",
     "        p = partial(correct_module, max_cells, do_ff, index_v, CHUNK_SIZE, total_sequences, sequences_qm,\n",
     "                   bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range, max_pulses, dbparms,\n",
-    "                   fileparms, nodb, no_non_linear_corrections)\n",
+    "                   fileparms, nodb, no_non_linear_corrections, mark_non_lin_region, linear_between, nlc_version)\n",
     "        \n",
     "        r = view.map_sync(p, inp)\n",
     "        #r = list(map(p, inp))\n",
-- 
GitLab


From e3df964d4c50225ed13c7c25e3f5ee83be5d282a Mon Sep 17 00:00:00 2001
From: Steffen Hauf <steffen.hauf@xfel.eu>
Date: Thu, 5 Sep 2019 15:20:18 +0200
Subject: [PATCH 2/5] Add enum changes

---
 cal_tools/cal_tools/enums.py | 41 ++++++++++++++++++------------------
 cal_tools/cal_tools/tools.py |  1 -
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/cal_tools/cal_tools/enums.py b/cal_tools/cal_tools/enums.py
index 0e30568a4..ed7a0457f 100644
--- a/cal_tools/cal_tools/enums.py
+++ b/cal_tools/cal_tools/enums.py
@@ -5,24 +5,25 @@ class BadPixels(Enum):
     """ The European XFEL Bad Pixel Encoding
     """
         
-    OFFSET_OUT_OF_THRESHOLD  = 0b00000000000000000001 # bit 1
-    NOISE_OUT_OF_THRESHOLD   = 0b00000000000000000010 # bit 2
-    OFFSET_NOISE_EVAL_ERROR  = 0b00000000000000000100 # bit 3
-    NO_DARK_DATA             = 0b00000000000000001000 # bit 4
-    CI_GAIN_OF_OF_THRESHOLD  = 0b00000000000000010000 # bit 5
-    CI_LINEAR_DEVIATION      = 0b00000000000000100000 # bit 6
-    CI_EVAL_ERROR            = 0b00000000000001000000 # bit 7
-    FF_GAIN_EVAL_ERROR       = 0b00000000000010000000 # bit 8
-    FF_GAIN_DEVIATION        = 0b00000000000100000000 # bit 9
-    FF_NO_ENTRIES            = 0b00000000001000000000 # bit 10
-    CI2_EVAL_ERROR           = 0b00000000010000000000 # bit 11
-    VALUE_IS_NAN             = 0b00000000100000000000 # bit 12
-    VALUE_OUT_OF_RANGE       = 0b00000001000000000000 # bit 13
-    GAIN_THRESHOLDING_ERROR  = 0b00000010000000000000 # bit 14
-    DATA_STD_IS_ZERO         = 0b00000100000000000000 # bit 15
-    ASIC_STD_BELOW_NOISE     = 0b00001000000000000000 # bit 16
-    INTERPOLATED             = 0b00010000000000000000 # bit 17
-    NOISY_ADC                = 0b00100000000000000000 # bit 18
-    OVERSCAN                 = 0b01000000000000000000 # bit 19
-    NON_SENSITIVE            = 0b10000000000000000000 # bit 20
+    OFFSET_OUT_OF_THRESHOLD  = 0b000000000000000000001 # bit 1
+    NOISE_OUT_OF_THRESHOLD   = 0b000000000000000000010 # bit 2
+    OFFSET_NOISE_EVAL_ERROR  = 0b000000000000000000100 # bit 3
+    NO_DARK_DATA             = 0b000000000000000001000 # bit 4
+    CI_GAIN_OF_OF_THRESHOLD  = 0b000000000000000010000 # bit 5
+    CI_LINEAR_DEVIATION      = 0b000000000000000100000 # bit 6
+    CI_EVAL_ERROR            = 0b000000000000001000000 # bit 7
+    FF_GAIN_EVAL_ERROR       = 0b000000000000010000000 # bit 8
+    FF_GAIN_DEVIATION        = 0b000000000000100000000 # bit 9
+    FF_NO_ENTRIES            = 0b000000000001000000000 # bit 10
+    CI2_EVAL_ERROR           = 0b000000000010000000000 # bit 11
+    VALUE_IS_NAN             = 0b000000000100000000000 # bit 12
+    VALUE_OUT_OF_RANGE       = 0b000000001000000000000 # bit 13
+    GAIN_THRESHOLDING_ERROR  = 0b000000010000000000000 # bit 14
+    DATA_STD_IS_ZERO         = 0b000000100000000000000 # bit 15
+    ASIC_STD_BELOW_NOISE     = 0b000001000000000000000 # bit 16
+    INTERPOLATED             = 0b000010000000000000000 # bit 17
+    NOISY_ADC                = 0b000100000000000000000 # bit 18
+    OVERSCAN                 = 0b001000000000000000000 # bit 19
+    NON_SENSITIVE            = 0b010000000000000000000 # bit 20
+    NON_LIN_RESPONSE_REGION  = 0b100000000000000000000 # bit 21
 
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index 3ca11ac74..8036b7ac4 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -487,7 +487,6 @@ def get_from_db(device, constant, condition, empty_constant,
     from iCalibrationDB import ConstantMetaData, Versions
     import zmq
     
-    timeout=1200000
     if version_info:
         meta_only = False
     
-- 
GitLab


From 5d3bb3149acbe9ae7744d74f5628c6d570b8918e Mon Sep 17 00:00:00 2001
From: Steffen Hauf <steffen.hauf@xfel.eu>
Date: Thu, 5 Sep 2019 15:22:30 +0200
Subject: [PATCH 3/5] Remove not needed print

---
 cal_tools/cal_tools/lpdlib.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/cal_tools/cal_tools/lpdlib.py b/cal_tools/cal_tools/lpdlib.py
index fff0d401c..01997ad41 100644
--- a/cal_tools/cal_tools/lpdlib.py
+++ b/cal_tools/cal_tools/lpdlib.py
@@ -265,10 +265,8 @@ class LpdCorrections:
                 msk[midx] = BadPixels.NON_LIN_RESPONSE_REGION.value
                 numnonlin = np.count_nonzero(midx, axis=(1,2))
                 nlf += numnonlin
-            print(nlf.shape)
             nlf = nlf/float(im.shape[0] * im.shape[1])
-            print("Marked non-linear-regions")
-
+            
         # hacky way of smoothening transition region between med and low
         
         cfac = 1
-- 
GitLab


From 4339343af9cd8acaa5b5ea5be624d5232637ea2a Mon Sep 17 00:00:00 2001
From: Steffen Hauf <steffen.hauf@xfel.eu>
Date: Thu, 5 Sep 2019 15:51:05 +0200
Subject: [PATCH 4/5] Update LPD_Correct_and_Verify.ipynb

---
 notebooks/LPD/LPD_Correct_and_Verify.ipynb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/notebooks/LPD/LPD_Correct_and_Verify.ipynb b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
index acee0ff45..d09bc106d 100644
--- a/notebooks/LPD/LPD_Correct_and_Verify.ipynb
+++ b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
@@ -48,9 +48,9 @@
     "sequences_per_node = 1 # sequence files to process per node\n",
     "timeout_cal_db = 30000 # timeout for calibration db requests in milliseconds\n",
     "dont_mark_non_lin_region = False # do not mark non-linear regions in BP map\n",
-    "linear_between_high_gain = [-5000, 2500]  # region in which high gain is considered linear\n",
-    "linear_between_med_gain = [300, 3000]  # region in which medium gain is considered linear\n",
-    "linear_between_low_gain = [300, 3000]  # region in which low gain is considered linear\n",
+    "linear_between_high_gain = [-5000, 2500]  # region in which high gain is considered linear, in ADU\n",
+    "linear_between_med_gain = [300, 3000]  # region in which medium gain is considered linear, in ADU\n",
+    "linear_between_low_gain = [300, 3000]  # region in which low gain is considered linear, in ADU\n",
     "nlc_version = 2 # version of NLC to use\n",
     "\n",
     "def balance_sequences(in_folder, run, sequences, sequences_per_node):\n",
-- 
GitLab


From 905a1cd1b386c4c0ecee0470bc9494a8935c69a8 Mon Sep 17 00:00:00 2001
From: Steffen Hauf <steffen.hauf@xfel.eu>
Date: Thu, 5 Sep 2019 15:51:58 +0200
Subject: [PATCH 5/5] Update lpdlib.py

---
 cal_tools/cal_tools/lpdlib.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cal_tools/cal_tools/lpdlib.py b/cal_tools/cal_tools/lpdlib.py
index 01997ad41..7b9b89865 100644
--- a/cal_tools/cal_tools/lpdlib.py
+++ b/cal_tools/cal_tools/lpdlib.py
@@ -258,7 +258,7 @@ class LpdCorrections:
         im -= og
         
         nlf = 0
-        if self.mark_nonlin:
+        if self.mark_nonlin and self.linear_between is not None:
             for gl, lr in enumerate(self.linear_between):
                 
                 midx = (gain == gl) & ((im < lr[0]) | (im > lr[1]))
-- 
GitLab