diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 691a3c41836fcdcb04fd14d35a5990c82a05a019..558ec084bdc7f843f6f653a7f6b44cb4a88afc90 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -1,20 +1,23 @@
import copy
from enum import Enum
+import os
import h5py
import numpy as np
-from cal_tools.enums import BadPixels
-from cal_tools.tools import get_constant_from_db, get_constant_from_db_and_time
-from iCalibrationDB import Constants, Conditions, Detectors
from scipy.signal import cwt, ricker
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import StandardScaler
+from cal_tools.agipdutils import assemble_constant_dict
+from cal_tools.enums import BadPixels
+from cal_tools.tools import get_constant_from_db, get_constant_from_db_and_time
+from iCalibrationDB import Constants, Conditions, Detectors
+
def get_num_cells(fname, loc, module):
with h5py.File(fname, "r") as f:
cells = \
- f["INSTRUMENT/{}/DET/{}CH0:xtdf/image/cellId".format(loc, module)][()]
+ f[f"INSTRUMENT/{loc}/DET/{module}CH0:xtdf/image/cellId"][()]
if cells.shape[0] == 0:
return None
maxcell = np.max(cells)
@@ -27,8 +30,7 @@ def get_acq_rate(fname, loc, module):
with h5py.File(fname, "r") as f:
try:
pulses = \
- np.squeeze(f["INSTRUMENT/{}/DET/{}CH0:xtdf/image/pulseId".format(
- loc, module)][:2])
+ np.squeeze(f[f"INSTRUMENT/{loc}/DET/{module}CH0:xtdf/image/pulseId"][:2]) # noqa
diff = pulses[1] - pulses[0]
except:
diff = 0
@@ -60,7 +62,7 @@ def get_gain_setting(fname, h5path_ctrl):
if (setupr == 0 and pattern_type_idx < 4) or (
setupr == 32 and pattern_type_idx == 4):
return 0
- elif(setupr == 8 and pattern_type_idx < 4) or (
+ elif (setupr == 8 and pattern_type_idx < 4) or (
setupr == 40 and pattern_type_idx == 4):
return 1
else:
@@ -204,6 +206,13 @@ class AgipdCorrections:
raise Exception('Correction Booleans: {} are not available!'
.format(list(set(corr_bools) - set(tot_corr_bools))))
+ # Flags allowing for pulse capacitor constant retrieval.
+ self.pc_bools = [self.corr_bools.get("rel_gain"),
+ self.corr_bools.get("adjust_mg_baseline"),
+ self.corr_bools.get('blc_noise'),
+ self.corr_bools.get('blc_hmatch'),
+ self.corr_bools.get('blc_stripes'),
+ self.melt_snow]
def get_iteration_range(self):
"""Returns a range expression over which to iterate in chunks
"""
@@ -373,7 +382,7 @@ class AgipdCorrections:
sh_idxs = self.get_shadowed_stripe(dd, 30, 0.95)
# collect all shadowed regions excluding double pixels
- idx = []
+ idx = []
for sh_idx in sh_idxs:
if len(sh_idx)>2:
idx += sh_idx
@@ -1020,9 +1029,9 @@ class AgipdCorrections:
# before doing relative gain correction we need to evaluate any
# baseline shifts
# as they are effectively and additional offset in the data
- if (self.corr_bools.get('blc_noise') or
- self.corr_bools.get('blc_hmatch') or
- self.corr_bools.get('blc_stripes') ):
+ if (self.corr_bools.get('blc_noise') or
+ self.corr_bools.get('blc_hmatch') or
+ self.corr_bools.get('blc_stripes')):
# do this image wise, as the shift is per image
for i in range(im.shape[0]):
@@ -1463,67 +1472,43 @@ class AgipdCorrections:
(self.low_edges, self.high_edges, self.signal_edges,
self.dig_signal_edges))
- def initialize_from_db(self, dbparms, qm, only_dark=False):
- """ Initialize calibration constants from the calibration database
-
- :param dbparms: a tuple containing relevant database parameters,
- can be either:
- * cal_db_interface, creation_time, max_cells_db, bias_voltage,
- photon_energy in which case the db timeout is set to 300 seconds,
- the cells to query dark image derived constants from the
- database is set to the global value
-
- * cal_db_interface, creation_time, max_cells_db, bias_voltage,
- photon_energy, max_cells_db_dark here the number of memory
- cells to query dark derived data differs from the global
- value, the db timeout is set to 300 seconds
-
- * cal_db_interface, creation_time, max_cells_db, bias_voltage,
- photon_energy, max_cells_db_dark, timeout_cal_db
- additionally a timeout is given
-
- :param qm: quadrant and module of the constants to load in Q1M1
- notation
- :param only_dark: load only dark image derived constants. This
- implies that a `calfile` is used to load the remaining
- constants. Useful to reduce DB traffic and interactions
- for non-frequently changing constants, i.e. such which are
- not usually updated during a beamtime.
-
- The `cal_db_interface` parameter in the `dbparms` tuple may be in
- one of the following notations:
- * tcp://host:port to directly identify the host and port to
- connect to
- * tcp://host:port_low#port_high to specify a port range from
- which a random port will be picked. E.g. specifying
-
- tcp://max-exfl016:8015#8025
-
- will randomly pick an address in the range max-exfl016:8015 and
- max-exfl016:8025.
-
- The latter notation allows for load-balancing.
-
- This routine loads the following constants as given in
- `iCalibrationDB`:
-
- Dark Image Derived
- ------------------
-
- * Constants.AGIPD.Offset
- * Constants.AGIPD.Noise
- * Constants.AGIPD.BadPixelsDark
- * Constants.AGIPD.ThresholdsDark
-
- Pulse Capacitor Derived
- -----------------------
-
- * Constants.AGIPD.SlopesPC
-
- Flat-Field Derived
-
- * Constants.AGIPD.SlopesFF
+ def retrieve_constant_and_time(self, const_dict, device,
+ cal_db_interface, creation_time):
+ """
+ A wrapper around get_constant_from_db_and_time to get
+ constant image and creation time using a dictionary of
+ the names of the required constant from the data base
+ and their parameter operating conditions.
+
+ :param const_dict: (Dict) A dictionary with constants to retrieve
+ and their operating conditions.
+ e.g.{ "Offset": ["zeros", (128, 512, memory_cells, 3), "Dark"]
+ "Dark-cond": {Condition-name}: condition-value }
+ :param device: (iCalibrationDB obj) Object for AGIPD's device
+ :param cal_db_interface: (Str) The port interface for
+ calibrationDBRemote
+ :param creation_time: (Str) Raw data creation time
+ :return:
+ cons_data: (np.array) Constant data
+ when: (Dict) A dictionary with all retrieved constants
+ and their creation-time. {"Constant-Name": "creation-time"}
+ """
+ when = {}
+ cons_data = {}
+ for cname, cval in const_dict.items():
+ condition = \
+ getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])
+ cons_data[cname], when[cname] = \
+ get_constant_from_db_and_time(device,
+ getattr(Constants.AGIPD, cname)(), # noqa
+ condition,
+ getattr(np, cval[0])(cval[1]),
+ cal_db_interface,
+ creation_time)
+ return cons_data, when
+ def init_constants(self, cons_data, only_dark):
+ """
For CI derived gain, a mean multiplication factor of 4.48 compared
to medium gain is used, as no reliable CI data for all memory cells
exists of the current AGIPD instances.
@@ -1566,142 +1551,23 @@ class AgipdCorrections:
medium gain = ff * rfpc
low gain = medium gain / 4.48
+ :param cons_data: A dictionary for each retrieved constant value.
+ :param only_dark: A flag for retrieving only dark constants.
+ :return:
"""
- if len(dbparms) == 5:
- (cal_db_interface, creation_time, max_cells_db,
- bias_voltage, photon_energy) = dbparms
- max_cells_db_dark = max_cells_db
- timeout_cal_db = 30000
- elif len(dbparms) == 6:
- (cal_db_interface, creation_time, max_cells_db,
- bias_voltage, photon_energy, max_cells_db_dark) = dbparms
+ bpixels = cons_data["BadPixelsDark"].astype(np.uint32)
- if max_cells_db_dark is None:
- max_cells_db_dark = max_cells_db
- timeout_cal_db = 30000
- else:
- (cal_db_interface, creation_time, max_cells_db,
- bias_voltage, photon_energy, max_cells_db_dark,
- timeout_cal_db) = dbparms
-
- if max_cells_db_dark is None:
- max_cells_db_dark = max_cells_db
-
- # if "#" in cal_db_interface:
- # prot, serv, ran = cal_db_interface.split(":")
- # r1, r2 = ran.split("#")
- # cal_db_interface = ":".join(
- # [prot, serv, str(np.random.randint(int(r1), int(r2)))])
-
- dinstance = getattr(Detectors, self.cal_det_instance)
- when = {}
- if self.corr_bools.get('only_offset'):
- only_dark = True
-
- offset, when['offset'] = \
- get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.Offset(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db_dark,
- bias_voltage=bias_voltage,
- gain_setting=self.gain_setting,
- acquisition_rate=self.acquisition_rate), # noqa
- np.zeros((128, 512,
- max_cells_db,
- 3)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
-
- noise, when['noise'] = \
- get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.Noise(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db_dark,
- bias_voltage=bias_voltage,
- gain_setting=self.gain_setting,
- acquisition_rate=self.acquisition_rate), # noqa
- np.zeros(
- (128, 512, max_cells_db, 3)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
-
- bpixels, when['bpixels'] = \
- get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.BadPixelsDark(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db_dark,
- bias_voltage=bias_voltage,
- gain_setting=self.gain_setting,
- acquisition_rate=self.acquisition_rate), # noqa
- np.zeros(
- (128, 512, max_cells_db, 3)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
- bpixels = bpixels.astype(np.uint32)
-
- thresholds, when['thresholds'] = \
- get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.ThresholdsDark(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db_dark,
- bias_voltage=bias_voltage,
- gain_setting=self.gain_setting,
- acquisition_rate=self.acquisition_rate), # noqa
- np.ones((128, 512, max_cells_db, 2)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
-
- # we have all we need for dark image derived constants
- # initialize and return
- if only_dark:
- self.initialize(offset=offset, rel_gain=None, mask=bpixels,
- noise=noise, thresholds=thresholds)
- return when
+ if self.corr_bools.get('only_offset') or only_dark:
+ self.initialize(offset=cons_data["Offset"],
+ rel_gain=None, mask=bpixels,
+ noise=cons_data["Noise"],
+ thresholds=cons_data["ThresholdsDark"])
+ return
if self.corr_bools.get("xray_corr"):
- bpff, when['bpff'] = \
- get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.BadPixelsFF(),
- Conditions.Illuminated.AGIPD(
- max_cells_db,
- bias_voltage,
- photon_energy,
- pixels_x=512,
- pixels_y=128,
- beam_energy=None,
- gain_setting=self.gain_setting,
- acquisition_rate=self.acquisition_rate),
- # noqa
- np.zeros(
- (128, 512, max_cells_db)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
-
- bpixels |= bpff.astype(np.uint32)[..., :bpixels.shape[2], None]
-
- slopesFF, when['slopesFF'] = \
- get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.SlopesFF(),
- Conditions.Illuminated.AGIPD(
- max_cells_db,
- bias_voltage,
- photon_energy,
- pixels_x=512,
- pixels_y=128,
- beam_energy=None,
- gain_setting=self.gain_setting,
- acquisition_rate=self.acquisition_rate), # noqa
- np.ones((128, 512, max_cells_db, 2)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
- slopesFF = np.squeeze(slopesFF)
+ bpixels |= cons_data["BadPixelsFF"].astype(np.uint32)[..., :bpixels.shape[2], None] # noqa
+ slopesFF = np.squeeze(cons_data["SlopesFF"])
if len(slopesFF.shape) == 4:
slopesFF = slopesFF[..., 0]
@@ -1722,46 +1588,12 @@ class AgipdCorrections:
xray_cor /= np.nanmedian(xray_cor)
else:
xray_cor = None
-
- if (self.corr_bools.get("rel_gain") or
- self.corr_bools.get("adjust_mg_baseline") or
- self.corr_bools.get('blc_noise') or
- self.corr_bools.get('blc_hmatch') or
- self.corr_bools.get('blc_stripes') or
- self.melt_snow is not False):
-
- # add additonal bad pixel information
- bppc, when['bppc'] = \
- get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.BadPixelsPC(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db,
- bias_voltage=bias_voltage,
- gain_setting=self.gain_setting,
- acquisition_rate=self.acquisition_rate),
- # noqa
- np.zeros(
- (max_cells_db, 128, 512)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
- bppc = np.moveaxis(bppc.astype(np.uint32), 0, 2)
+ # add additional bad pixel information
+ if any(self.pc_bools):
+ bppc = np.moveaxis(cons_data["BadPixelsPC"].astype(np.uint32), 0, 2)
bpixels |= bppc[..., :bpixels.shape[2], None]
- # load also non-dark constants from the database
- slopesPC, when['slopesPC'] = \
- get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.SlopesPC(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db,
- bias_voltage=bias_voltage,
- acquisition_rate=self.acquisition_rate), # noqa
- np.ones(
- (128, 512, max_cells_db, 10)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
- slopesPC = slopesPC.astype(np.float32)
+ slopesPC = cons_data["SlopesPC"].astype(np.float32)
# this will handle some historical data in a slighly different format
# constant dimension injected first
@@ -1795,9 +1627,133 @@ class AgipdCorrections:
rel_gain = None
frac_high_med = None
- self.initialize(offset, rel_gain, xray_cor, bpixels, noise, thresholds,
+ self.initialize(cons_data["Offset"], rel_gain, xray_cor, bpixels,
+ cons_data["Noise"],
+ cons_data["ThresholdsDark"],
frac_high_med=frac_high_med,
md_additional_offset=md_additional_offset)
+ return
+
+ def initialize_from_yaml(self, const_yaml, qm, only_dark=False):
+ """
+ Initialize calibration constants from a yaml file
+
+ :param const_yaml: (Dict) fromed from a yaml file in pre-notebook,
+ which consists of metadata of either the constant file path or the
+ empty constant shape, and the creation-time of the retrieved constants
+ :param qm: Module's virtual name (e.g. Q1M1)
+ :param only_dark: A flag for retrieving only dark constants.
+ :return:
+ """
+
+ device = getattr(getattr(Detectors, self.cal_det_instance), qm)
+
+ # string of the device name.
+ dname = device.device_name
+ cons_data = dict()
+ when = dict()
+ for cname, mdata in const_yaml[dname].items():
+ when[cname] = mdata["creation-time"]
+ if when[cname]:
+ cf = h5py.File(mdata["file-path"])
+ cons_data[cname] = np.copy(cf[f"{dname}/{cname}/0/data"])
+ else:
+ # Create empty constant using the list elements
+ cons_data[cname] = \
+ getattr(np, mdata["file-path"][0])(mdata["file-path"][1]) # noqa
+
+ self.init_constants(cons_data, only_dark)
+
+ return when
+
+ def initialize_from_db(self, dbparms, qm, only_dark=False):
+ """ Initialize calibration constants from the calibration database
+
+ :param dbparms: a tuple containing relevant database parameters,
+ can be either:
+ * cal_db_interface, creation_time, max_cells_db, bias_voltage,
+ photon_energy in which case the db timeout is set to 300 seconds,
+ the cells to query dark image derived constants from the
+ database is set to the global value
+
+ * cal_db_interface, creation_time, max_cells_db, bias_voltage,
+ photon_energy, max_cells_db_dark, timeout_cal_db
+ additionally a timeout is given
+
+ :param qm: quadrant and module of the constants to load in Q1M1
+ notation
+ :param only_dark: load only dark image derived constants. This
+ implies that a `calfile` is used to load the remaining
+ constants. Useful to reduce DB traffic and interactions
+ for non-frequently changing constants, i.e. such which are
+ not usually updated during a beamtime.
+
+ The `cal_db_interface` parameter in the `dbparms` tuple may be in
+ one of the following notations:
+ * tcp://host:port to directly identify the host and port to
+ connect to
+ * tcp://host:port_low#port_high to specify a port range from
+ which a random port will be picked. E.g. specifying
+
+ tcp://max-exfl016:8015#8025
+
+ will randomly pick an address in the range max-exfl016:8015 and
+ max-exfl016:8025.
+
+ The latter notation allows for load-balancing.
+
+ This routine loads the following constants as given in
+ `iCalibrationDB`:
+
+ Dark Image Derived
+ ------------------
+
+ * Constants.AGIPD.Offset
+ * Constants.AGIPD.Noise
+ * Constants.AGIPD.BadPixelsDark
+ * Constants.AGIPD.ThresholdsDark
+
+ Pulse Capacitor Derived
+ -----------------------
+
+ * Constants.AGIPD.SlopesPC
+
+ Flat-Field Derived
+
+ * Constants.AGIPD.SlopesFF
+
+ """
+ max_cells_db_dark = None
+
+ # dbparms has a length of 5 in
+ # playground/QuickCorrect-SingleModule.ipynb
+ if len(dbparms) == 5:
+ (cal_db_interface, creation_time, max_cells_db,
+ bias_voltage, photon_energy) = dbparms
+ else:
+ (cal_db_interface, creation_time, max_cells_db,
+ bias_voltage, photon_energy, max_cells_db_dark,
+ timeout_cal_db) = dbparms
+
+ if max_cells_db_dark is None:
+ max_cells_db_dark = max_cells_db
+
+ # Device object for retrieving constants
+ device = getattr(getattr(Detectors, self.cal_det_instance), qm)
+
+ const_dict = \
+ assemble_constant_dict(self.corr_bools, self.pc_bools, max_cells_db_dark,
+ bias_voltage, self.gain_setting,
+ self.acquisition_rate, photon_energy,
+ beam_energy=None, only_dark=only_dark)
+
+ cons_data, when =\
+ self.retrieve_constant_and_time(const_dict, device,
+ cal_db_interface,
+ creation_time)
+
+ self.init_constants(cons_data, only_dark)
+
return when
def initialize_from_file(self, filename, qm, with_dark=True):
diff --git a/cal_tools/cal_tools/agipdutils.py b/cal_tools/cal_tools/agipdutils.py
new file mode 100644
index 0000000000000000000000000000000000000000..7322580a04757fe23714dbc7ca728dc762c4011e
--- /dev/null
+++ b/cal_tools/cal_tools/agipdutils.py
@@ -0,0 +1,67 @@
+import h5py
+import numpy as np
+
+
+def assemble_constant_dict(corr_bools, pc_bools, memory_cells, bias_voltage,
+ gain_setting, acquisition_rate,
+ photon_energy, beam_energy=None, only_dark=False):
+ """
+ Assemble a dictionary with the iCalibrationDB constant names and
+ the operating conditions for retrieveing the required constants
+ for correction.
+
+ :param corr_bools: (Dict) A dict of booleans for applying
+ specific corrections
+ :param pc_bools: (List) A list of booleans to enable SlopesPC retrieval
+ :param memory_cells: (Int) Number of memory cells
+ :param bias_voltage: (Int) Bias Voltage
+ :param gain_setting: (Float) Gain setting
+ :param acquisition_rate: (Float) Acquisition rate
+ :param photon_energy: (Float) Photong energy
+ :param beam_energy: (Float) Beam Energy
+ :param only_dark: (Bool) Indicating a retrieval for dark
+ constants only from db
+ :return: const_dict: (Dict) An assembeld dictionary that can be used
+ to retrieve the required constants
+ """
+
+ darkcond = [
+ "Dark", {
+ "memory_cells": memory_cells,
+ "bias_voltage": bias_voltage,
+ "acquisition_rate": acquisition_rate,
+ "gain_setting": gain_setting,
+ "pixels_x": 512,
+ "pixels_y": 128, }
+ ]
+ const_dict = {
+ "Offset": ["zeros", (128, 512, memory_cells, 3), darkcond],
+ "Noise": ["zeros", (128, 512, memory_cells, 3), darkcond],
+ "ThresholdsDark": ["ones", (128, 512, memory_cells, 2), darkcond],
+ "BadPixelsDark": ["zeros", (128, 512, memory_cells, 3), darkcond],
+ }
+
+ if not (corr_bools.get('only_offset') or only_dark):
+
+ if any(pc_bools):
+ const_dict["BadPixelsPC"] = \
+ ["zeros", (memory_cells, 128, 512), darkcond]
+ const_dict["SlopesPC"] = \
+ ["ones", (128, 512, memory_cells, 10), darkcond]
+
+ if corr_bools.get("xray_corr"):
+ # Add illuminated conditions
+ illumcond = [
+ "Illuminated", {
+ "beam_energy": beam_energy,
+ "photon_energy": photon_energy
+ }
+ ]
+ illumcond[1].update(darkcond[1])
+
+ const_dict["BadPixelsFF"] = ["zeros", (128, 512, memory_cells),
+ illumcond]
+ const_dict["SlopesFF"] = ["ones", (128, 512, memory_cells, 2),
+ illumcond]
+
+ return const_dict
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 69c91d7de01bf7fa5769fc9282a0a2949effb9a9..d0c8b178e465972ce86aade7d26f3d8502732e70 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -128,27 +128,30 @@
},
"outputs": [],
"source": [
- "import sys\n",
"from collections import OrderedDict\n",
- "\n",
- "# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
+ "from datetime import timedelta\n",
"import os\n",
+ "import sys\n",
+ "\n",
+ "from dateutil import parser\n",
"import h5py\n",
"import numpy as np\n",
"import matplotlib\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.pyplot as plt\n",
"from ipyparallel import Client\n",
- "print(f\"Connecting to profile {cluster_profile}\")\n",
- "view = Client(profile=cluster_profile)[:]\n",
- "view.use_dill()\n",
+ "import yaml\n",
"\n",
"from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
"from cal_tools.tools import (map_modules_from_folder, parse_runs, run_prop_seq_from_path, get_notebook_name,\n",
- " get_dir_creation_date, get_constant_from_db)\n",
+ " get_dir_creation_date, get_constant_from_db)\n",
+ "\n",
"from cal_tools.agipdlib import get_gain_setting\n",
- "from dateutil import parser\n",
- "from datetime import timedelta\n",
+ "\n",
+ "# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
+ "print(f\"Connecting to profile {cluster_profile}\")\n",
+ "view = Client(profile=cluster_profile)[:]\n",
+ "view.use_dill()\n",
"\n",
"il_mode = interlaced\n",
"max_cells = mem_cells\n",
@@ -195,13 +198,6 @@
" dinstance = \"AGIPD1M2\"\n"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
{
"cell_type": "code",
"execution_count": null,
@@ -366,7 +362,7 @@
"end_time": "2019-02-21T11:30:16.057429Z",
"start_time": "2019-02-21T11:30:10.082114Z"
},
- "scrolled": true
+ "scrolled": false
},
"outputs": [],
"source": [
@@ -376,17 +372,19 @@
" bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range,\n",
" bins_dig_gain_vs_signal, max_pulses, dbparms, fileparms, nodb, chunk_size_idim,\n",
" special_opts, il_mode, loc, dinstance, force_hg_if_below, force_mg_if_below,\n",
- " mask_noisy_adc, acq_rate, gain_setting, corr_bools, h5path, h5path_idx, inp):\n",
+ " mask_noisy_adc, acq_rate, gain_setting, corr_bools, h5path, h5path_idx, const_yaml, inp):\n",
" print(\"foo\")\n",
" import numpy as np\n",
" import copy\n",
" import h5py\n",
" import socket\n",
+ " import traceback\n",
" from datetime import datetime\n",
" import re\n",
" import os\n",
" from influxdb import InfluxDBClient\n",
" import subprocess\n",
+ " from iCalibrationDB import Constants, Conditions, Detectors\n",
" from cal_tools.enums import BadPixels\n",
" from cal_tools.agipdlib import AgipdCorrections, SnowResolution\n",
" from cal_tools.agipdlib import get_num_cells, get_acq_rate\n",
@@ -441,11 +439,10 @@
" reason = \"\"\n",
" filename, filename_out, channel, qm = inp\n",
" print(\"Have input\")\n",
- " \n",
" if max_cells == 0:\n",
" max_cells = get_num_cells(filename, loc, channel)\n",
" if max_cells is None:\n",
- " raise ValueError(\"No raw images found for {}\".format(qm))\n",
+ " raise ValueError(f\"No raw images found for {qm}\")\n",
" else:\n",
" cells = np.arange(max_cells)\n",
" \n",
@@ -459,8 +456,8 @@
" if dbparms[5] == 0:\n",
" dbparms[5] = dbparms[2]\n",
"\n",
- " print(\"Set memory cells to {}\".format(max_cells))\n",
- " print(\"Set acquistion rate cells to {} MHz\".format(acq_rate))\n",
+ " print(f\"Set memory cells to {max_cells}\")\n",
+ " print(f\"Set acquistion rate cells to {acq_rate} MHz\")\n",
"\n",
" # AGIPD correction requires path without the leading \"/\"\n",
" if h5path[0] == '/':\n",
@@ -494,8 +491,18 @@
" agipd_corr.get_valid_image_idx()\n",
" except IOError:\n",
" return\n",
+ " device = getattr(getattr(Detectors, dinstance), qm)\n",
+ " \n",
+ " # check if there is a yaml file in out_folder that has the device constants.\n",
" if not nodb:\n",
- " when = agipd_corr.initialize_from_db(dbparms, qm, only_dark=(fileparms != \"\"))\n",
+ " if const_yaml and device.device_name in const_yaml:\n",
+ " print(fileparms != \"\")\n",
+ " agipd_corr.initialize_from_yaml(const_yaml, qm,\n",
+ " only_dark=((fileparms != \"\")))\n",
+ " else:\n",
+ " when = agipd_corr.initialize_from_db(dbparms, qm,\n",
+ " only_dark=(fileparms != \"\"))\n",
+ "\n",
" if fileparms != \"\" and not corr_bools[\"only_offset\"]:\n",
" agipd_corr.initialize_from_file(fileparms, qm, with_dark=nodb)\n",
" print(\"Initialized constants\")\n",
@@ -503,7 +510,6 @@
" for irange in agipd_corr.get_iteration_range():\n",
" agipd_corr.correct_agipd(irange)\n",
" print(\"Iterated\")\n",
- "\n",
" print(\"All iterations are finished\")\n",
" hists, edges = agipd_corr.get_histograms()\n",
" hists_signal_low, hists_signal_high, hists_gain_vs_signal, hists_dig_gain_vs_signal, hist_pulses = hists\n",
@@ -516,8 +522,8 @@
" print(\"Closed files\")\n",
" \n",
" except Exception as e:\n",
- " print(e)\n",
- " err = e\n",
+ " err = f\"Error: {e}\\nError traceback: {traceback.format_exc()}\"\n",
+ " print(err)\n",
" success = False\n",
" reason = \"Error\"\n",
" \n",
@@ -530,7 +536,7 @@
" #influx = create_influx_entry(run, proposal, qm, sequence, filesize, CHUNK_SIZE, total_sequences, success, duration, reason)\n",
" #client.write_points([influx])\n",
" return (hists_signal_low, hists_signal_high, hists_gain_vs_signal, hists_dig_gain_vs_signal, hist_pulses,\n",
- " low_edges, high_edges, signal_edges, dig_signal_edges, gain_stats, max_cells, when, qm, err)\n",
+ " low_edges, high_edges, signal_edges, dig_signal_edges, gain_stats, max_cells, acq_rate, when, qm, err)\n",
" \n",
"done = False\n",
"first_files = []\n",
@@ -568,17 +574,25 @@
"all_cells = []\n",
"whens = []\n",
"errors = []\n",
+ "const_yaml = None\n",
+ "\n",
+ "if os.path.isfile(f'{out_folder}/retrieved_constants.yml'):\n",
+ " with open(f'{out_folder}/retrieved_constants.yml', \"r\") as f:\n",
+ " const_yaml = yaml.load(f.read(), Loader=yaml.FullLoader)\n",
+ "\n",
+ "mod_dev = dict()\n",
"while not done:\n",
- " \n",
" dones = []\n",
" first = True\n",
- " for i in range(16):\n",
- " qm = \"Q{}M{}\".format(i//4 +1, i % 4 + 1)\n",
+ " for i in range(len(modules)):\n",
+ " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" fname_in = str(mapped_files[qm].get())\n",
" dones.append(mapped_files[qm].empty())\n",
+ " device = getattr(getattr(Detectors, dinstance), qm)\n",
+ " mod_dev[qm] = device.device_name\n",
" else:\n",
- " print(\"Skipping {}\".format(qm))\n",
+ " print(f\"Skipping {qm}\")\n",
" first_files.append((None, None))\n",
" continue\n",
" fout = os.path.abspath(\"{}/{}\".format(out_folder, (os.path.split(fname_in)[-1]).replace(\"RAW\", \"CORR\")))\n",
@@ -586,16 +600,17 @@
" first_files.append((fname_in, fout))\n",
" inp.append((fname_in, fout, i, qm))\n",
" first = False\n",
+ " \n",
+ "\n",
" if len(inp) >= min(MAX_PAR, left):\n",
- " print(\"Running {} tasks parallel\".format(len(inp)))\n",
+ " print(f\"Running {len(inp)} tasks parallel\")\n",
" p = partial(correct_module, max_cells, index_v, CHUNK_SIZE, total_sequences,\n",
" sequences_qm, bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range,\n",
" bins_dig_gain_vs_signal, max_pulses, dbparms, fileparms, nodb, chunk_size_idim,\n",
" special_opts, il_mode, karabo_id, dinstance, force_hg_if_below, force_mg_if_below,\n",
- " mask_noisy_adc, acq_rate, gain_setting, corr_bools, h5path, h5path_idx)\n",
+ " mask_noisy_adc, acq_rate, gain_setting, corr_bools, h5path, h5path_idx, const_yaml)\n",
"\n",
" r = view.map_sync(p, inp)\n",
- "\n",
" #r = list(map(p, inp))\n",
"\n",
" inp = []\n",
@@ -604,7 +619,7 @@
" init_hist = False\n",
" for rr in r:\n",
" if rr is not None:\n",
- " hl, hh, hg, hdg, hp, low_edges, high_edges, signal_edges, dig_signal_edges, gs, cells, when, qm, err = rr\n",
+ " hl, hh, hg, hdg, hp, low_edges, high_edges, signal_edges, dig_signal_edges, gs, cells, acq_rate, when, qm, err = rr\n",
" all_cells.append(cells)\n",
" whens.append((qm, when))\n",
" errors.append(err)\n",
@@ -620,44 +635,64 @@
" hists_dig_gain_vs_signal += hdg.astype(np.float64)\n",
" gain_stats += gs\n",
" \n",
- " done = all(dones)"
+ " done = all(dones)\n",
+ "\n",
+ "print(f\"Corrected raw data of {cells} memory cells and {acq_rate} MHz acquisition rate\")"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "scrolled": false
+ },
"outputs": [],
"source": [
- "print(\"Constants were injected on: \")\n",
+ "# if there is a yml file that means a leading notebook got processed\n",
+ "# and the reporting would be generated from it.\n",
+ "fst_print = True\n",
+ "\n",
"to_store = []\n",
+ "line = []\n",
"for i, (qm, when) in enumerate(whens):\n",
- " print(qm)\n",
- " line = [qm]\n",
- " # If correction is crashed\n",
- " if errors[i] is not None:\n",
+ " # expose errors while applying correction\n",
+ " if errors[i]:\n",
" print(\"Error: {}\".format(errors[i]) )\n",
- " else:\n",
- " for key, item in when.items():\n",
- " if hasattr(item, 'strftime'):\n",
- " item = item.strftime('%y-%m-%d %H:%M')\n",
- " when[key] = item\n",
- " print('{:.<12s}'.format(key), item)\n",
- " \n",
- " # Store few time stamps if exists\n",
- " # Add NA to keep array structure\n",
- " for key in ['offset', 'slopesPC', 'slopesFF']:\n",
- " if when and key in when:\n",
- " line.append(when[key])\n",
- " else:\n",
- " if errors[i] is not None:\n",
- " line.append('Err')\n",
- " else:\n",
- " line.append('NA')\n",
- " to_store.append(line)\n",
"\n",
- "np.savetxt(\"{}/tmp_const_beginat_S{:05d}.csv\".format(out_folder, sequences[0]), \n",
- " np.array(to_store).astype(str), fmt='%s', delimiter = ',')"
+ " if not const_yaml or mod_dev[qm] not in const_yaml:\n",
+ " if fst_print:\n",
+ " print(\"Constants are retrieved with creation time: \")\n",
+ " fst_print = False\n",
+ " \n",
+ " line = [qm]\n",
+ "\n",
+ " # If correction is crashed\n",
+ " if not errors[i]:\n",
+ " print(f\"{qm}:\")\n",
+ " for key, item in when.items():\n",
+ " if hasattr(item, 'strftime'):\n",
+ " item = item.strftime('%y-%m-%d %H:%M')\n",
+ " when[key] = item\n",
+ " print('{:.<12s}'.format(key), item)\n",
+ "\n",
+ " # Store few time stamps if exists\n",
+ " # Add NA to keep array structure\n",
+ " for key in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
+ " if when and key in when and when[key]:\n",
+ " line.append(when[key])\n",
+ " else:\n",
+ " if errors[i] is not None:\n",
+ " line.append('Err')\n",
+ " else:\n",
+ " line.append('NA')\n",
+ "\n",
+ " if len(line) > 0:\n",
+ " to_store.append(line)\n",
+ "\n",
+ "seq = sequences[0] if sequences else 0\n",
+ "if len(to_store) > 0:\n",
+ " with open(f\"{out_folder}/retrieved_constants_s{seq}.yml\",\"w\") as fyml:\n",
+ " yaml.safe_dump({\"time-summary\": {f\"S{seq}\":to_store}}, fyml)"
]
},
{
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
index 7fb5ed4748dda8eb326c8339fb2e29071e704ce4..b38add62968d46d7a31145f6d160ca5ba0315618 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
@@ -14,8 +14,9 @@
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "run = 119 # runs to process, required\n",
- "out_folder = \"/gpfs/exfel/exp/SPB/201802/p002157/usr/test/test5/r0119\" # path to output to, required"
+ "run = 11 # runs to process, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_Corr\" # path to output to, required\n",
+ "modules = [-1]"
]
},
{
@@ -24,11 +25,13 @@
"metadata": {},
"outputs": [],
"source": [
+ "import os\n",
+ "\n",
"import dateutil.parser\n",
"import glob\n",
"import numpy as np\n",
"import re\n",
- "import os\n",
+ "import yaml\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
@@ -44,28 +47,48 @@
"metadata": {},
"outputs": [],
"source": [
- "# load temporary data from csv files\n",
- "fnames = sorted(glob.glob('{}/tmp_const_beginat_*'.format(out_folder)))\n",
+ "if os.path.isfile(f'{out_folder}/retrieved_constants.yml'):\n",
+ " with open(f\"{out_folder}/retrieved_constants.yml\",\"r\") as fyml:\n",
+ " main_dict = yaml.load(fyml)\n",
+ "else:\n",
+ " main_dict = {\"time-summary\":dict()}\n",
"\n",
+ "if modules[0] == -1:\n",
+ " modules = list(range(16))\n",
+ "\n",
+ "# This is needed only if AGIPD Correction notebook had no precorrection notebooks for retrieving constants\n",
+ "# gather all generated sequence yml files for time summary of retrieved constant in retrieved_constants.yml\n",
+ "fnames = sorted(glob.glob(f'{out_folder}/retrieved_constants_*yml')) \n",
+ "for f in fnames:\n",
+ " with open(f,\"r\") as fyml:\n",
+ " fdict = yaml.load(fyml)\n",
+ " # append different sequences's time summary to the main yaml\n",
+ " for k, v in fdict[\"time-summary\"].items():\n",
+ " main_dict[\"time-summary\"][k] = v\n",
+ " os.remove(f)\n",
+ "\n",
+ "with open(f\"{out_folder}/retrieved_constants.yml\",\"w\") as fyml:\n",
+ " yaml.safe_dump(main_dict, fyml)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with open(f\"{out_folder}/retrieved_constants.yml\",\"r\") as fyml:\n",
+ " time_summary = yaml.load(fyml)\n",
+ "# check if pre-notebook has retrieved constants for all modules.\n",
"const_times = []\n",
"seq = []\n",
- "for fname in fnames:\n",
- " # read sequence number and array of timestamps\n",
- " s = int(re.findall(r'.*S([0-9]{5}).*', fname)[0])\n",
- " arr = np.loadtxt(fname, dtype=str, delimiter = ',')\n",
- " # job can run across several sequences\n",
- " # number of modules is fixed to 16\n",
- " if arr.shape[0]>16:\n",
- " seq += range(int(s), int(s)+arr.shape[0]//16)\n",
- " arr = arr.reshape(arr.shape[0]//16, 16, arr.shape[1])\n",
- " const_times = const_times + list(arr)\n",
- " \n",
- " else:\n",
- " const_times.append(arr)\n",
- " seq.append(s)\n",
- " os.remove(fname)\n",
- "\n",
- "const_times = np.array(const_times)\n"
+ "for k, v in sorted(time_summary[\"time-summary\"].items()):\n",
+ " arr = np.array(v)\n",
+ " arr = arr.reshape(arr.shape[0]//len(modules), len(modules), arr.shape[1])\n",
+ " const_times = const_times + list(arr)\n",
+ " seq.append(k)\n",
+ " \n",
+ "const_times = np.array(const_times)"
]
},
{
@@ -76,20 +99,24 @@
"source": [
"# Function print summary of constant injection time\n",
"# To reduce printouts only unique entries are shown.\n",
- "def plot_const_table(const, pos):\n",
- " print(\"{} were injected on: \".format(const))\n",
+ "def const_table(const, pos):\n",
+ " \"\"\"\n",
+ " Create a summary table for the creation time differences for\n",
+ " the retrieved constants (Offset, SlopesPC, SlopesFF).\n",
+ " \"\"\"\n",
+ " print(f\"{const} were injected on: \")\n",
"\n",
+ " # catch timing difference in retrieve constants\n",
" unique, idx, counts = np.unique(const_times[:,:,pos], return_inverse=True, return_counts=True)\n",
- " idx = idx.reshape((const_times.shape[0],16))\n",
+ " idx = idx.reshape((const_times.shape[0], len(modules)))\n",
" \n",
" table = []\n",
- "\n",
" for i in range(0, counts.shape[0]):\n",
" line = [ const_times[:,:,pos][idx==i][0] ]\n",
" mods = ''\n",
" for i_s, s in enumerate(seq):\n",
- " if( const_times[i_s,:,0][idx[i_s]==i].shape[0]>0 ):\n",
- " mods = mods+ 'S{}: {}, '.format(seq[i_s], const_times[i_s,:,0][idx[i_s]==i])\n",
+ " if(const_times[i_s,:,0][idx[i_s]==i].shape[0] > 0):\n",
+ " mods = mods+ '{}: {}, '.format(s, const_times[i_s,:,0][idx[i_s]==i])\n",
" line.append(mods)\n",
" table.append(line)\n",
"\n",
@@ -98,13 +125,19 @@
" else:\n",
" table[np.argmax(counts)][1] = 'Rest of the modules'\n",
"\n",
- " md = display(Latex(tabulate.tabulate(table, tablefmt='latex', \n",
- " headers=[\"Time stamps\", \"Modules and sequences\"]))) \n",
- " \n",
- "for i_key, key in enumerate(['offset', 'slopesPC', 'slopesFF']):\n",
- " if const_times.shape[2]>i_key+1:\n",
- " plot_const_table(key, i_key+1)\n"
+ " md = display(Latex(tabulate.tabulate(table, tablefmt='latex',\n",
+ " headers=[\"Time stamps\", \"Modules and sequences\"])))\n",
+ "for i_key, key in enumerate(['Offset', 'SlopesPC', 'SlopesFF']):\n",
+ " if const_times.shape[2] > i_key+1:\n",
+ " const_table(key, i_key+1)"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..13ebd016af104dc0ce815bd2ee8926d1ee606cc6
--- /dev/null
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -0,0 +1,461 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# AGIPD Retrieving Constants Pre-correction #\n",
+ "\n",
+ "Author: K.Ahmed, Version: 1.0\n",
+ "\n",
+ "Retrieving Required Constants for Offline Calibration of the AGIPD Detector"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2019-02-21T11:30:06.730220Z",
+ "start_time": "2019-02-21T11:30:06.658286Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "cluster_profile = \"noDB\"\n",
+ "in_folder = \"/gpfs/exfel/exp/MID/201931/p900107/raw\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_Corr\" # the folder to output to, required\n",
+ "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
+ "modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
+ "run = 11 # runs to process, required\n",
+ "\n",
+ "karabo_id = \"MID_DET_AGIPD1M-1\" # karabo karabo_id\n",
+ "karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
+ "path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
+ "h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP_TEST' # path to control information\n",
+ "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
+ "karabo_da_control = 'DA02' # karabo DA for control infromation\n",
+ "\n",
+ "use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
+ "cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
+ "creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
+ "\n",
+ "calfile = \"\" # path to calibration file. Leave empty if all data should come from DB\n",
+ "nodb = False # if set only file-based constants will be used\n",
+ "mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
+ "bias_voltage = 300\n",
+ "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
+ "gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
+ "photon_energy = 9.2 # photon energy in keV\n",
+ "max_cells_db_dark = 0 # set to a value different than 0 to use this value for dark data DB queries\n",
+ "max_cells_db = 0 # set to a value different than 0 to use this value for DB queries\n",
+ "\n",
+ "# Correction Booleans\n",
+ "only_offset = False # Apply only Offset correction. if False, Offset is applied by Default. if True, Offset is only applied.\n",
+ "rel_gain = False # do relative gain correction based on PC data\n",
+ "xray_gain = True # do relative gain correction based on xray data\n",
+ "blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
+ "blc_stripes = False # if set, baseline corrected via stripes\n",
+ "blc_hmatch = False # if set, base line correction via histogram matching is attempted \n",
+ "match_asics = False # if set, inner ASIC borders are matched to the same signal level\n",
+ "adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Fill dictionaries comprising bools and arguments for correction and data analysis\n",
+ "\n",
+ "# Here the herarichy and dependability for correction booleans are defined \n",
+ "corr_bools = {}\n",
+ "\n",
+ "# offset is at the bottom of AGIPD correction pyramid.\n",
+ "corr_bools[\"only_offset\"] = only_offset\n",
+ "\n",
+ "# Dont apply any corrections if only_offset is requested \n",
+ "if not only_offset:\n",
+ " corr_bools[\"adjust_mg_baseline\"] = adjust_mg_baseline\n",
+ " corr_bools[\"rel_gain\"] = rel_gain\n",
+ " corr_bools[\"xray_corr\"] = xray_gain\n",
+ " corr_bools[\"blc_noise\"] = blc_noise\n",
+ " corr_bools[\"blc_hmatch\"] = blc_hmatch"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import sys\n",
+ "from collections import OrderedDict\n",
+ "\n",
+ "# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
+ "import os\n",
+ "import h5py\n",
+ "import numpy as np\n",
+ "import matplotlib\n",
+ "matplotlib.use(\"agg\")\n",
+ "import matplotlib.pyplot as plt\n",
+ "from ipyparallel import Client\n",
+ "print(f\"Connecting to profile {cluster_profile}\")\n",
+ "view = Client(profile=cluster_profile)[:]\n",
+ "view.use_dill()\n",
+ "\n",
+ "from iCalibrationDB import Constants, Conditions, Detectors\n",
+ "from cal_tools.tools import (map_modules_from_folder, get_dir_creation_date)\n",
+ "from cal_tools.agipdlib import get_gain_setting\n",
+ "from dateutil import parser\n",
+ "from datetime import timedelta"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2019-02-21T11:30:07.086286Z",
+ "start_time": "2019-02-21T11:30:06.929722Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "max_cells = mem_cells\n",
+ "\n",
+ "creation_time = None\n",
+ "if use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run)\n",
+ " offset = parser.parse(creation_date_offset)\n",
+ " delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
+ " creation_time += delta\n",
+ " print(f\"Using {creation_time} as creation time\")\n",
+ "\n",
+ "if sequences[0] == -1:\n",
+ " sequences = None\n",
+ "\n",
+ "if in_folder[-1] == \"/\":\n",
+ " in_folder = in_folder[:-1]\n",
+ "print(f\"Outputting to {out_folder}\")\n",
+ "\n",
+ "os.makedirs(out_folder, exist_ok=True)\n",
+ "\n",
+ "import warnings\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "from cal_tools.agipdlib import SnowResolution\n",
+ "\n",
+ "melt_snow = False if corr_bools[\"only_offset\"] else SnowResolution.NONE"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "control_fname = f'{in_folder}/RAW-R{run:04d}-{karabo_da_control}-S00000.h5'\n",
+ "\n",
+ "if gain_setting == 0.1:\n",
+ " if creation_time.replace(tzinfo=None) < parser.parse('2020-01-31'):\n",
+ " print(\"Set gain-setting to None for runs taken before 2020-01-31\")\n",
+ " gain_setting = None\n",
+ " else:\n",
+ " try:\n",
+ " gain_setting = get_gain_setting(control_fname, h5path_ctrl)\n",
+ " except Exception as e:\n",
+ " print(f'Error while reading gain setting: {e}\\n')\n",
+ " \n",
+ "print(f\"Gain setting: {gain_setting}\")\n",
+ "print(f\"Detector in use is {karabo_id}\")\n",
+ "\n",
+ "if karabo_da[0] == '-1':\n",
+ " if modules[0] == -1:\n",
+ " modules = list(range(16))\n",
+ " karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
+ "else:\n",
+ " modules = [int(x[-2:]) for x in karabo_da]\n",
+ "\n",
+ "\n",
+ "h5path_ctrl = h5path_ctrl.format(karabo_id_control)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2019-02-21T11:30:07.974174Z",
+ "start_time": "2019-02-21T11:30:07.914832Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# set everything up filewise\n",
+ "print(f\"Checking the files before retrieving constants\")\n",
+ "mmf = map_modules_from_folder(in_folder, run, path_template, karabo_da, sequences)\n",
+ "\n",
+ "mapped_files, mod_ids, total_sequences, sequences_qm, _ = mmf"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Retrieve Constants ##"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from functools import partial\n",
+ "import yaml\n",
+ "\n",
+ "def retrieve_constants(karabo_id, bias_voltage, max_cells, acq_rate, \n",
+ " gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
+ " cal_db_interface, creation_time, \n",
+ " corr_bools, pc_bools, inp):\n",
+ " \"\"\"\n",
+ " Retreive constant for each module in parallel and produce a dictionary\n",
+ " with the creation-time and constant file path.\n",
+ " \n",
+ " :param karabo_id: (STR) Karabo ID\n",
+ " :param bias_voltage: (FLOAT) Bias Voltage\n",
+ " :param max_cells: (INT) Memory cells\n",
+ " :param acq_rate: (FLOAT) Acquisition Rate\n",
+ " :param gain_setting: (FLOAT) Gain setting\n",
+ " :param photon_energy: (FLOAT) Photon Energy\n",
+ " :param only_dark: (BOOL) only retrieve dark constants\n",
+ " :param nodb_with_dark: (BOOL) no constant retrieval even for dark\n",
+ " :param cal_db_interface: (STR) the database interface port\n",
+ " :param creation_time: (STR) raw data creation time\n",
+ " :param corr_bools: (DICT) A dictionary with bools for applying requested corrections\n",
+ " :param pc_bools: (LIST) list of bools to retrieve pulse capacitor constants\n",
+ " :param inp: (LIST) input for the parallel cluster of the partial function\n",
+ " :return:\n",
+ " mdata_dict: (DICT) dictionary with the metadata for the retrieved constants\n",
+ " dev.device_name: (STR) device name\n",
+ " \"\"\"\n",
+ "\n",
+ " import numpy as np\n",
+ " import sys\n",
+ " import traceback\n",
+ " \n",
+ " from cal_tools.agipdlib import get_num_cells, get_acq_rate\n",
+ " from cal_tools.agipdutils import assemble_constant_dict\n",
+ " from cal_tools.tools import get_from_db\n",
+ "\n",
+ " from iCalibrationDB import Constants, Conditions, Detectors\n",
+ "\n",
+ " err = None\n",
+ "\n",
+ " qm_files, qm, dev, idx = inp\n",
+ " # get number of memory cells from a sequence file with image data\n",
+ " for f in qm_files:\n",
+ " if max_cells == 0:\n",
+ " max_cells = get_num_cells(f, karabo_id, idx)\n",
+ " if max_cells is None:\n",
+ " if f != qm_files[-1]:\n",
+ " continue\n",
+ " else:\n",
+ " raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
+ " else:\n",
+ " cells = np.arange(max_cells)\n",
+ " # get out of the loop,\n",
+ " # if max_cells is successfully calculated. \n",
+ " break\n",
+ "\n",
+ " if acq_rate == 0.:\n",
+ " acq_rate = get_acq_rate(f, karabo_id, idx)\n",
+ " else:\n",
+ " acq_rate = None\n",
+ "\n",
+ " print(f\"Set memory cells to {max_cells}\")\n",
+ " print(f\"Set acquistion rate cells to {acq_rate} MHz\")\n",
+ "\n",
+ " # avoid creating retireving constant, if requested.\n",
+ " if not nodb_with_dark: \n",
+ " const_dict = assemble_constant_dict(corr_bools, pc_bools, max_cells, bias_voltage,\n",
+ " gain_setting, acq_rate, photon_energy,\n",
+ " beam_energy=None, only_dark=only_dark)\n",
+ "\n",
+ " # Retrieve multiple constants through an input dictionary\n",
+ " # to return a dict of useful metadata.\n",
+ " mdata_dict = dict()\n",
+ " for cname, cval in const_dict.items():\n",
+ " try:\n",
+ " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
+ " co, mdata = \\\n",
+ " get_from_db(dev, getattr(Constants.AGIPD, cname)(),\n",
+ " condition, getattr(np, cval[0])(cval[1]),\n",
+ " cal_db_interface, creation_time, meta_only=True)\n",
+ " mdata_const = mdata.calibration_constant_version\n",
+ " # saving metadata in a dict\n",
+ " mdata_dict[cname] = dict()\n",
+ " # check if constant was sucessfully retrieved.\n",
+ " if mdata.comm_db_success: \n",
+ " mdata_dict[cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
+ " f\"{mdata_const.filename}\"\n",
+ " mdata_dict[cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " else:\n",
+ " mdata_dict[cname][\"file-path\"] = const_dict[cname]\n",
+ " mdata_dict[cname][\"creation-time\"] = None\n",
+ " except Exception as e:\n",
+ " err = f\"Error: {e}, Traceback: {traceback.format_exc()}\"\n",
+ " print(err)\n",
+ "\n",
+ " return qm, mdata_dict, dev.device_name, acq_rate, max_cells, err\n",
+ "\n",
+ "pc_bools = [corr_bools.get(\"rel_gain\"),\n",
+ " corr_bools.get(\"adjust_mg_baseline\"),\n",
+ " corr_bools.get('blc_noise'),\n",
+ " corr_bools.get('blc_hmatch'),\n",
+ " corr_bools.get('blc_stripes'),\n",
+ " melt_snow]\n",
+ "\n",
+ "# Extracting Instrument string\n",
+ "instrument = karabo_id.split(\"_\")[0]\n",
+ "if instrument == \"SPB\":\n",
+ " dinstance = \"AGIPD1M1\"\n",
+ "else:\n",
+ " dinstance = \"AGIPD1M2\"\n",
+ "\n",
+ "print(f\"Instrument {instrument}\")\n",
+ "print(f\"Detector instance {dinstance}\")\n",
+ "\n",
+ "inp = []\n",
+ "only_dark = False\n",
+ "nodb_with_dark = False\n",
+ "if not nodb:\n",
+ " only_dark=(calfile != \"\")\n",
+ "if calfile != \"\" and not corr_bools[\"only_offset\"]:\n",
+ " nodb_with_dark = nodb\n",
+ "\n",
+ "# A dict to connect virtual device\n",
+ "# to actual device name.\n",
+ "for i in range(len(modules)):\n",
+ " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " if qm in mapped_files and not mapped_files[qm].empty():\n",
+ " device = getattr(getattr(Detectors, dinstance), qm)\n",
+ " qm_files = [str(mapped_files[qm].get()) for _ in range(mapped_files[qm].qsize())]\n",
+ "\n",
+ " else:\n",
+ " print(f\"Skipping {qm}\")\n",
+ " continue\n",
+ "\n",
+ " inp.append((qm_files, qm, device, i))\n",
+ "\n",
+ "p = partial(retrieve_constants, karabo_id, bias_voltage, max_cells, \n",
+ " acq_rate, gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
+ " cal_db_interface, creation_time, \n",
+ " corr_bools, pc_bools)\n",
+ "\n",
+ "results = view.map_sync(p, inp)\n",
+ "#results = list(map(p, inp))\n",
+ "mod_dev = dict()\n",
+ "mdata_dict = dict()\n",
+ "for r in results:\n",
+ " if r:\n",
+ " qm, md_dict, dname, acq_rate, max_cells, err = r\n",
+ " mod_dev[dname] = {\"mod\": qm, \"err\": err}\n",
+ " if err:\n",
+ " print(f\"Error for module {qm}: {err}\")\n",
+ " mdata_dict[dname] = md_dict\n",
+ "# check if it is requested not to retrieve any constants from the database\n",
+ "if not nodb_with_dark:\n",
+ " with open(f\"{out_folder}/retrieved_constants.yml\", \"w\") as outfile:\n",
+ " yaml.dump(mdata_dict, outfile)\n",
+ " print(\"\\nRetrieved constants for modules: \",\n",
+ " f\"{[', '.join([f'Q{x//4+1}M{x%4+1}' for x in modules])]}\")\n",
+ " print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {max_cells}\\n\"\n",
+ " f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Photon Energy: {photon_energy}\\n\")\n",
+ " print(f\"Constant metadata is saved in retrieved_constants.yml\\n\")\n",
+ "else:\n",
+ " print(\"No constants were retrieved as calibrated files will be used.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(\"Constants are retrieved with creation time: \")\n",
+ "i = 0\n",
+ "when = dict()\n",
+ "to_store = []\n",
+ "\n",
+ "for dname, dinfo in mod_dev.items():\n",
+ " print(dinfo[\"mod\"], \":\")\n",
+ " line = [dinfo[\"mod\"]]\n",
+ " if dname in mdata_dict:\n",
+ " for cname, mdata in mdata_dict[dname].items():\n",
+ " if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
+ " mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
+ " print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
+ " # Store few time stamps if exists\n",
+ " # Add NA to keep array structure\n",
+ " for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
+ " if not dname in mdata_dict or dinfo[\"err\"]:\n",
+ " line.append('Err')\n",
+ " else:\n",
+ " if cname in mdata_dict[dname]:\n",
+ " if mdata_dict[dname][cname][\"creation-time\"]:\n",
+ " line.append(mdata_dict[dname][cname][\"creation-time\"])\n",
+ " else:\n",
+ " line.append('NA')\n",
+ " else:\n",
+ " line.append('NA')\n",
+ " to_store.append(line)\n",
+ "\n",
+ " i += 1\n",
+ " if sequences:\n",
+ " seq_num = sequences[0]\n",
+ " else:\n",
+ " # if sequences[0] changed to None as it was -1\n",
+ " seq_num = 0\n",
+ " \n",
+ "with open(f\"{out_folder}/retrieved_constants.yml\",\"r\") as fyml:\n",
+ " time_summary = yaml.load(fyml)\n",
+ " time_summary.update({\"time-summary\": {\n",
+ " \"SAll\":to_store\n",
+ " }})\n",
+ "with open(f\"{out_folder}/retrieved_constants.yml\",\"w\") as fyml:\n",
+ " yaml.safe_dump(time_summary, fyml)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index fd4e7a3c37fff97bb166b28e089da01ef0727a09..6a9e40efde73e4ccb8c0beb115ad856a292cf0b3 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -423,7 +423,7 @@ elif len(sys.argv) >= 3:
func = get_notebook_function(nb, ext_func)
if func is None:
- warnings.warn("Didn't find concurrency function {} in notebook".format(ext_func),
+ warnings.warn(f"Didn't find concurrency function {ext_func} in notebook",
RuntimeWarning)
else:
@@ -528,7 +528,7 @@ def create_finalize_script(fmt_args, temp_path, job_list):
echo 'Running finalize script'
python3 -c "from xfel_calibrate.finalize import finalize;
finalize(joblist={{joblist}},
- finaljob=$1,
+ finaljob=$1,
run_path='{{run_path}}',
out_path='{{out_path}}',
project='{{project}}',
@@ -567,13 +567,12 @@ def save_executed_command(run_tmp_path, version):
finfile.write(' '.join(sys.argv))
-def get_launcher_command(args, temp_path, dependent, job_list):
+def get_launcher_command(args, temp_path, dep_jids):
"""
Return a slurm launcher command
:param args: Command line arguments
:param temp_path: Temporary path to run job
- :param dependent: True if job is dependent
- :param job_list: A list of dependent jobs
+ :param dep_jids: A list of dependent jobs
:return: List of commands and parameters to be used by subprocess
"""
@@ -604,9 +603,9 @@ def get_launcher_command(args, temp_path, dependent, job_list):
launcher_slurm += " --mem {}G".format(args.get('slurm_mem', '500'))
- if dependent:
+ if len(dep_jids):
srun_dep = " --dependency=afterok"
- for jobid in job_list:
+ for jobid in dep_jids:
srun_dep += ":{}".format(jobid)
launcher_slurm += srun_dep
@@ -631,7 +630,7 @@ def remove_duplications(l):
def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
final_job=False, job_list=[], fmt_args={}, cluster_cores=8,
- sequential=False, dependent=False,
+ sequential=False, dep_jids=[],
show_title=True, cluster_profile='NO_CLUSTER'):
""" Launch a concurrent job on the cluster via SLURM
"""
@@ -666,7 +665,7 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
# then run an sbatch job
srun_base = []
if not sequential:
- srun_base = get_launcher_command(args, temp_path, dependent, job_list)
+ srun_base = get_launcher_command(args, temp_path, dep_jids)
print(" ".join(srun_base))
srun_base += [os.path.abspath("{}/bin/slurm_calibrate.sh".format(os.path.dirname(__file__))), # path to helper sh
@@ -773,6 +772,7 @@ def run():
print("Not running on cluster")
try:
+ pre_notebooks = notebooks[detector][caltype].get("pre_notebooks", [])
notebook = notebooks[detector][caltype]["notebook"]
dep_notebooks = notebooks[detector][caltype].get("dep_notebooks", [])
notebook = os.path.abspath(
@@ -795,7 +795,7 @@ def run():
func = get_notebook_function(nb, ext_func)
if func is None:
- warnings.warn("Didn't find concurrency function {} in notebook".format(ext_func),
+ warnings.warn(f"Didn't find concurrency function {ext_func} in notebook",
RuntimeWarning)
else:
# remove help calls as they will cause the argument parser to exit
@@ -888,29 +888,43 @@ def run():
}
joblist = []
- if concurrency.get("parameter", None) is None:
- cluster_cores = concurrency.get("cluster cores", 8)
+ cluster_cores = concurrency.get("cluster cores", 8)
+ # Check if there are pre-notebooks
+ for i, notebook in enumerate(pre_notebooks):
+ notebook_path = os.path.abspath(
+ "{}/{}".format(os.path.dirname(__file__), notebook))
+ with open(notebook_path, "r") as f:
+ lead_nb = nbformat.read(f, as_version=4)
+ jobid = concurrent_run(run_tmp_path, lead_nb,
+ os.path.basename(notebook),
+ args,
+ job_list=joblist, fmt_args=fmt_args,
+ cluster_cores=cluster_cores,
+ sequential=sequential,
+ cluster_profile=cluster_profile)
+ joblist.append(jobid)
+ if concurrency.get("parameter", None) is None:
jobid = concurrent_run(run_tmp_path, nb,
os.path.basename(notebook), args,
final_job=True, job_list=joblist,
fmt_args=fmt_args,
cluster_cores=cluster_cores,
sequential=sequential,
+ dep_jids=joblist,
cluster_profile=cluster_profile)
-
joblist.append(jobid)
else:
cvar = concurrency["parameter"]
cvals = args.get(cvar, None)
- cluster_cores = concurrency.get("cluster cores", 8)
con_func = concurrency.get("use function", None)
# Consider [-1] as None
if cvals is None or cvals == [-1]:
defcval = concurrency.get("default concurrency", None)
if defcval is not None:
- print("Concurrency parameter '{}' is taken from notebooks.py".format(cvar))
+ print(f"Concurrency parameter '{cvar}' "
+ f"is taken from notebooks.py")
if not isinstance(defcval, (list, tuple)):
cvals = range(defcval)
else:
@@ -919,8 +933,8 @@ def run():
if cvals is None:
defcval = get_par_attr(parms, cvar, 'value')
if defcval is not None:
- print("Concurrency parameter '{}' is taken from '{}'".format(
- cvar, notebook))
+ print(f"Concurrency parameter '{cvar}' "
+ f"is taken from '{notebook}'")
if not isinstance(defcval, (list, tuple)):
cvals = [defcval]
else:
@@ -929,7 +943,7 @@ def run():
if con_func:
func = get_notebook_function(nb, con_func)
if func is None:
- warnings.warn("Didn't find concurrency function {} in notebook".format(con_func),
+ warnings.warn(f"Didn't find concurrency function {con_func} in notebook",
RuntimeWarning)
else:
df = {}
@@ -944,12 +958,13 @@ def run():
for arg in sig.parameters:
callargs.append(args[arg])
cvals = f(*callargs)
- print("Split concurrency into {}".format(cvals))
+ print(f"Split concurrency into {cvals}")
# get expected type
cvtype = get_par_attr(parms, cvar, 'type', list)
cvals = remove_duplications(cvals)
+ jlist = []
for cnum, cval in enumerate(cvals):
show_title = cnum == 0
# Job is not final if there are dependent notebooks
@@ -962,9 +977,10 @@ def run():
cluster_cores=cluster_cores,
sequential=sequential,
show_title=show_title,
+ dep_jids=joblist,
cluster_profile=cluster_profile)
- joblist.append(jobid)
-
+ jlist.append(jobid)
+ joblist.extend(jlist)
# Run dependent notebooks
for i, notebook in enumerate(dep_notebooks):
notebook_path = os.path.abspath(
@@ -975,11 +991,11 @@ def run():
jobid = concurrent_run(run_tmp_path, nb,
os.path.basename(notebook),
args,
+ dep_jids=joblist,
final_job=final_job,
job_list=joblist, fmt_args=fmt_args,
cluster_cores=cluster_cores,
sequential=sequential,
- dependent=True,
cluster_profile=cluster_profile)
joblist.append(jobid)
diff --git a/xfel_calibrate/finalize.py b/xfel_calibrate/finalize.py
index 0a54fd134f255d630ebe95886ac0227cbe47c8ee..695eb293b710c4150934c6738a1c07201b3f6cf8 100644
--- a/xfel_calibrate/finalize.py
+++ b/xfel_calibrate/finalize.py
@@ -313,6 +313,11 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
for dtmp in temp_dirs:
rmtree(f'{dtmp}/')
+ # Archiving files in slurm_tmp
+ if os.path.isfile(f'{out_path}/retrieved_constants.yml'):
+ move(f'{out_path}/retrieved_constants.yml',
+ f"{tmp_path}")
+
# Moving temporary files to out-folder after successful execution
# This helps in keeping elements needed for re-producibility.
print(f"Moving temporary files to final location"
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index 12c97b96acbfe3b921b747f207f695a7b10f1955..494d3686fdc36fe2bbe7c02912d42e00a1ceac66 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -26,6 +26,7 @@ notebooks = {
"cluster cores": 16},
},
"CORRECT": {
+ "pre_notebooks": ["notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb"],
"notebook": "notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb",
"dep_notebooks": [
"notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb"],