Merge branch 'fix/Epix10K_dark' into 'master'

Fix: Variable name See merge request detectors/pycalibration!294

Merge branch 'fix/Epix10K_dark' into 'master'
Fix: Variable name See merge request detectors/pycalibration!294
d84ce4a5 · Mikhail Karnevskiy · e61bf7dd · 5d406fc0 · d84ce4a5
Commit d84ce4a5 authored 4 years ago by Mikhail Karnevskiy
--- a/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
+++ b/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
@@ -153,7 +153,7 @@
    "                                                               temperature_k))\n",
    "    print(\"Operated in vacuum: {} \".format(in_vacuum))\n",
    "    \n",
-    "os.makedirs(out_path, exist_ok=True)"
+    "os.makedirs(out_folder, exist_ok=True)"
   ]
  },
  {

 %% Cell type:markdown id: tags:

 # ePix10K Dark Characterization #

 Author: M. Karnevskiy, Version 1.0

 The following notebook provides dark image analysis of the ePix10K detector.

 Dark characterization evaluates offset and noise of the detector and gives information about bad pixels. Resulting maps are saved as .h5 files for a latter use and injected to the calibration DB.

 %% Cell type:code id: tags:

 ``` python
 cluster_profile = "noDB" # ipcluster profile to use
 in_folder = '/gpfs/exfel/exp/HED/201922/p002550/raw' # input folder, required
 out_folder = '/gpfs/exfel/exp/HED/201922/p002550/usr/dark/' # output folder, required
 sequence = 0 # sequence file to use
 run = 55 # which run to read data from, required

 karabo_id = "HED_IA1_EPIX10K-1" # karabo karabo_id
 karabo_da = "EPIX03"  # data aggregators
 receiver_id = "RECEIVER" # inset for receiver devices
 path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data
 h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data/image/pixels' # path in the HDF5 file to images
 h5path_t = '/INSTRUMENT/{}/DET/{}:daqOutput/data/backTemp'  # path to find temperature at
 h5path_cntrl = '/CONTROL/{}/DET'  # path to control data

 use_dir_creation_date = True
 cal_db_interface = "tcp://max-exfl016:8020" # calibration DB interface to use
 cal_db_timeout = 300000 # timeout on caldb requests
 db_output = False # Output constants to the calibration database
 local_output = True # output constants locally

 temp_limits = 5 # limit for parameter Operational temperature
 number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used
 db_module = 'ePix10K_M43' # detector karabo_id
 bias_voltage = 200 # bias voltage
 in_vacuum = False # detector operated in vacuum
 fix_temperature = 290. # fix temperature to this value
 ```

 %% Cell type:code id: tags:

 ``` python
 import XFELDetAna.xfelprofiler as xprof

 profiler = xprof.Profiler()
 profiler.disable()
 from XFELDetAna.util import env
 env.iprofile = cluster_profile

 import warnings
 warnings.filterwarnings('ignore')

 from XFELDetAna import xfelpycaltools as xcal
 from XFELDetAna import xfelpyanatools as xana
 from XFELDetAna.plotting.util import prettyPlotting

 prettyPlotting = True
 from XFELDetAna.xfelreaders import ChunkReader
 from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5
 from cal_tools.tools import get_dir_creation_date, save_const_to_h5, get_random_db_interface

 from iCalibrationDB import (ConstantMetaData, Constants, Conditions, Detectors,
                            Versions)
 from iCalibrationDB.detectors import DetectorTypes

 import numpy as np
 import os
 import h5py
 import matplotlib.pyplot as plt
 %matplotlib inline

 h5path = h5path.format(karabo_id, receiver_id)
 h5path_t = h5path_t.format(karabo_id, receiver_id)
 h5path_cntrl = h5path_cntrl.format(karabo_id)

 def nImagesOrLimit(nImages, limit):
    if limit == 0:
        return nImages
    else:
        return min(nImages, limit)
 ```

 %% Cell type:code id: tags:

 ``` python
 proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]
 file_loc = 'proposal:{} runs:{}'.format(proposal, run)

 x = 356  # rows of the ePix10K
 y = 384  # columns of the ePix10K

 ped_dir = "{}/r{:04d}".format(in_folder, run)
 fp_name = path_template.format(run, karabo_da).format(sequence)
 filename = '{}/{}'.format(ped_dir, fp_name)

 print("Reading data from: {}\n".format(filename))
 print("Run number: {}".format(run))
 print("HDF5 path: {}".format(h5path))
 if use_dir_creation_date:
    creation_time = get_dir_creation_date(in_folder, run)
    print("Using {} as creation time".format(creation_time.isoformat()))
 ```

 %% Cell type:code id: tags:

 ``` python
 sensorSize = [x, y]
 chunkSize = 100  #Number of images to read per chunk

 #Sensor area will be analysed according to blocksize
 blockSize = [sensorSize[0] // 2, sensorSize[1] // 2]
 xcal.defaultBlockSize = blockSize
 cpuCores = 4  #Specifies the number of running cpu cores
 memoryCells = 1  #No mamery cells

 #Specifies total number of images to proceed
 nImages = fastccdreaderh5.getDataSize(filename, h5path)[0]
 nImages = nImagesOrLimit(nImages, number_dark_frames)
 print("\nNumber of dark images to analyze: ", nImages)
 run_parallel = False

 with h5py.File(filename, 'r') as f:
    integration_time = int(f['{}/CONTROL/expTime/value'.format(h5path_cntrl)][0])
    gain_setting = int(f['{}/CONTROL/encodedGainSetting/value'.format(h5path_cntrl)][0])
    temperature = np.mean(f[h5path_t])/100.
    temperature_k = temperature + 273.15
    if fix_temperature != 0:
        temperature_k = fix_temperature
        print("Temperature is fixed!")
    print("Bias voltage is {} V".format(bias_voltage))
    print("Gain setting {}".format(gain_setting))
    print("Detector integration time is set to {}".format(integration_time))
    print("Mean temperature was {:0.2f} °C / {:0.2f} K".format(temperature,
                                                               temperature_k))
    print("Operated in vacuum: {} ".format(in_vacuum))

-os.makedirs(out_path, exist_ok=True)
+os.makedirs(out_folder, exist_ok=True)
 ```

 %% Cell type:code id: tags:

 ``` python
 reader = ChunkReader(filename, fastccdreaderh5.readData,
                     nImages, chunkSize,
                     path=h5path,
                     pixels_x=sensorSize[0],
                     pixels_y=sensorSize[1], )
 ```

 %% Cell type:code id: tags:

 ``` python
 noiseCal = xcal.NoiseCalculator(sensorSize, memoryCells,
                                cores=cpuCores, blockSize=blockSize,
                                parallel=run_parallel)
 histCalRaw = xcal.HistogramCalculator(sensorSize, bins=1000,
                                      range=[0, 10000], parallel=False,
                                      memoryCells=memoryCells,
                                      cores=cpuCores, blockSize=blockSize)
 ```

 %% Cell type:code id: tags:

 ``` python
 for data in reader.readChunks():
    data = np.bitwise_and(data.astype(np.uint16), 0b0011111111111111).astype(np.float32)
    dx = np.count_nonzero(data, axis=(0, 1))
    data = data[:, :, dx != 0]
    histCalRaw.fill(data)
    noiseCal.fill(data) #Fill calculators with data

 constant_maps = {}
 constant_maps['Offset'] = noiseCal.getOffset()  #Produce offset map
 constant_maps['Noise'] = noiseCal.get()  #Produce noise map

 noiseCal.reset()  #Reset noise calculator
 print("Initial maps were created")
 ```

 %% Cell type:code id: tags:

 ``` python
 #**************OFFSET MAP HISTOGRAM***********#
 ho, co = np.histogram(constant_maps['Offset'].flatten(), bins=700)

 do = {'x': co[:-1],
      'y': ho,
      'y_err': np.sqrt(ho[:]),
      'drawstyle': 'bars',
      'color': 'cornflowerblue',
      }

 fig = xana.simplePlot(do, figsize='1col', aspect=2,
                      x_label='Offset (ADU)',
                      y_label="Counts", y_log=True,
                      )

 #*****NOISE MAP HISTOGRAM FROM THE OFFSET CORRECTED DATA*******#
 hn, cn = np.histogram(constant_maps['Noise'].flatten(), bins=200)

 dn = {'x': cn[:-1],
      'y': hn,
      'y_err': np.sqrt(hn[:]),
      'drawstyle': 'bars',
      'color': 'cornflowerblue',
      }

 fig = xana.simplePlot(dn, figsize='1col', aspect=2,
                      x_label='Noise (ADU)',
                      y_label="Counts",
                      y_log=True)

 #**************HEAT MAPS*******************#
 fig = xana.heatmapPlot(constant_maps['Offset'][:, :, 0],
                       x_label='Columns', y_label='Rows',
                       lut_label='Offset (ADU)',
                       x_range=(0, y),
                       y_range=(0, x), vmin=1000, vmax=4000)

 fig = xana.heatmapPlot(constant_maps['Noise'][:, :, 0],
                       x_label='Columns', y_label='Rows',
                       lut_label='Noise (ADU)',
                       x_range=(0, y),
                       y_range=(0, x), vmax=2 * np.mean(constant_maps['Noise']))
 ```

 %% Cell type:code id: tags:

 ``` python
 # Save constants to DB
 dclass="ePix10K"
 cal_db_interface = get_random_db_interface(cal_db_interface)
 for const_name in constant_maps.keys():
    metadata = ConstantMetaData()
    det = getattr(Constants, dclass)
    const = getattr(det, const_name)()
    const.data = constant_maps[const_name].data

    metadata.calibration_constant = const

    # set the operating condition
    dcond = Conditions.Dark
    condition =  getattr(dcond, dclass)(bias_voltage=bias_voltage,
                                        integration_time=integration_time,
                                        temperature=temperature_k,
                                        in_vacuum=in_vacuum,
                                        gain_setting=gain_setting)

    for parm in condition.parameters:
        if parm.name == "Sensor Temperature":
            parm.lower_deviation = temp_limits
            parm.upper_deviation = temp_limits

    device = getattr(Detectors, db_module)

    metadata.detector_condition = condition

    # specify the a version for this constant
    if creation_time is None:
        metadata.calibration_constant_version = Versions.Now(device=device)
    else:
        metadata.calibration_constant_version = Versions.Timespan(device=device,
                                                                  start=creation_time)

    metadata.calibration_constant_version.raw_data_location = file_loc

    if db_output:
        try:
            metadata.send(cal_db_interface, timeout=cal_db_timeout)
            print("Inject {} constants from {}".format(const_name,
                                          metadata.calibration_constant_version.begin_at))
        except Exception as e:
            print(e)

    if local_output:
        save_const_to_h5(metadata, out_folder)
        print("Calibration constant {} is stored locally.".format(const))
 ```