Newer
Older
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# AGIPD Retrieving Constants Pre-correction #\n",
"\n",
"Author: European XFEL Detector Group, Version: 1.0\n",
"\n",
"Retrieving Required Constants for Offline Calibration of the AGIPD Detector"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-02-21T11:30:06.730220Z",
"start_time": "2019-02-21T11:30:06.658286Z"
}
},
"outputs": [],
"source": [
"cluster_profile = \"noDB\"\n",
"in_folder = \"/gpfs/exfel/exp/SPB/202030/p900119/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_\" # the folder to output to, required\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
"run = 80 # runs to process, required\n",
"karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
"path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
"h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP_TEST' # path to control information\n",
"karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
"karabo_da_control = 'AGIPD1MCTRL00' # karabo DA for control infromation\n",
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
"\n",
"use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
"creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
"\n",
"calfile = \"\" # path to calibration file. Leave empty if all data should come from DB\n",
"nodb = False # if set only file-based constants will be used\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
"bias_voltage = 300\n",
"acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
"gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
"photon_energy = 9.2 # photon energy in keV\n",
"max_cells_db_dark = 0 # set to a value different than 0 to use this value for dark data DB queries\n",
"max_cells_db = 0 # set to a value different than 0 to use this value for DB queries\n",
"\n",
"# Correction Booleans\n",
"only_offset = False # Apply only Offset correction. if False, Offset is applied by Default. if True, Offset is only applied.\n",
"rel_gain = False # do relative gain correction based on PC data\n",
"xray_gain = True # do relative gain correction based on xray data\n",
"blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
"blc_stripes = False # if set, baseline corrected via stripes\n",
"blc_hmatch = False # if set, base line correction via histogram matching is attempted \n",
"match_asics = False # if set, inner ASIC borders are matched to the same signal level\n",
"adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Fill dictionaries comprising bools and arguments for correction and data analysis\n",
"\n",
"# Here the herarichy and dependability for correction booleans are defined \n",
"corr_bools = {}\n",
"\n",
"# offset is at the bottom of AGIPD correction pyramid.\n",
"corr_bools[\"only_offset\"] = only_offset\n",
"\n",
"# Dont apply any corrections if only_offset is requested \n",
"if not only_offset:\n",
" corr_bools[\"adjust_mg_baseline\"] = adjust_mg_baseline\n",
" corr_bools[\"rel_gain\"] = rel_gain\n",
" corr_bools[\"xray_corr\"] = xray_gain\n",
" corr_bools[\"blc_noise\"] = blc_noise\n",
" corr_bools[\"blc_hmatch\"] = blc_hmatch"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"from collections import OrderedDict\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
"import os\n",
"import h5py\n",
"import numpy as np\n",
"import matplotlib\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.pyplot as plt\n",
"from ipyparallel import Client\n",
"print(f\"Connecting to profile {cluster_profile}\")\n",
"view = Client(profile=cluster_profile)[:]\n",
"view.use_dill()\n",
"\n",
"from iCalibrationDB import Constants, Conditions, Detectors\n",
"from cal_tools.tools import (map_modules_from_folder, get_dir_creation_date)\n",
"from cal_tools.agipdlib import get_gain_setting\n",
"from dateutil import parser\n",
"from datetime import timedelta"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-02-21T11:30:07.086286Z",
"start_time": "2019-02-21T11:30:06.929722Z"
}
},
"outputs": [],
"source": [
"max_cells = mem_cells\n",
"\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run)\n",
" offset = parser.parse(creation_date_offset)\n",
" delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
" creation_time += delta\n",
" print(f\"Using {creation_time} as creation time\")\n",
"\n",
"if sequences[0] == -1:\n",
" sequences = None\n",
"\n",
"if in_folder[-1] == \"/\":\n",
" in_folder = in_folder[:-1]\n",
"print(f\"Outputting to {out_folder}\")\n",
"\n",
"os.makedirs(out_folder, exist_ok=True)\n",
"\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"from cal_tools.agipdlib import SnowResolution\n",
"\n",
"melt_snow = False if corr_bools[\"only_offset\"] else SnowResolution.NONE"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"control_fname = f'{in_folder}/r{run:04d}/RAW-R{run:04d}-{karabo_da_control}-S00000.h5'\n",
"h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
"\n",
"if gain_setting == 0.1:\n",
" if creation_time.replace(tzinfo=None) < parser.parse('2020-01-31'):\n",
" print(\"Set gain-setting to None for runs taken before 2020-01-31\")\n",
" gain_setting = None\n",
" else:\n",
" try:\n",
" gain_setting = get_gain_setting(control_fname, h5path_ctrl)\n",
" except Exception as e:\n",
" print(f'ERROR: while reading gain setting from: \\n{control_fname}')\n",
" print(e)\n",
" print(\"Set gain setting to 0\")\n",
" gain_setting = 0\n",
"\n",
"print(f\"Gain setting: {gain_setting}\")\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"\n",
"\n",
"# Extracting Instrument string\n",
"instrument = karabo_id.split(\"_\")[0]\n",
"# Evaluate detector instance for mapping\n",
"if instrument == \"SPB\":\n",
" dinstance = \"AGIPD1M1\"\n",
" nmods = 16\n",
"elif instrument == \"MID\":\n",
" dinstance = \"AGIPD1M2\"\n",
" nmods = 16\n",
"# TODO: Remove DETLAB\n",
"elif instrument == \"HED\" or instrument == \"DETLAB\":\n",
" dinstance = \"AGIPD500K\"\n",
" nmods = 8\n",
"\n",
"print(f\"Instrument {instrument}\")\n",
"print(f\"Detector instance {dinstance}\")\n",
"\n",
"\n",
"if karabo_da[0] == '-1':\n",
" if modules[0] == -1:\n",
" modules = list(range(nmods))\n",
" karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]"
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-02-21T11:30:07.974174Z",
"start_time": "2019-02-21T11:30:07.914832Z"
}
},
"outputs": [],
"source": [
"# set everything up filewise\n",
"print(f\"Checking the files before retrieving constants\")\n",
"mmf = map_modules_from_folder(in_folder, run, path_template, karabo_da, sequences)\n",
"\n",
"mapped_files, mod_ids, total_sequences, sequences_qm, _ = mmf"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Retrieve Constants ##"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from functools import partial\n",
"import yaml\n",
"\n",
"def retrieve_constants(karabo_id, bias_voltage, max_cells, acq_rate, \n",
" gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
" cal_db_interface, creation_time, \n",
" corr_bools, pc_bools, inp):\n",
" \"\"\"\n",
" Retreive constant for each module in parallel and produce a dictionary\n",
" with the creation-time and constant file path.\n",
" \n",
" :param karabo_id: (STR) Karabo ID\n",
" :param bias_voltage: (FLOAT) Bias Voltage\n",
" :param max_cells: (INT) Memory cells\n",
" :param acq_rate: (FLOAT) Acquisition Rate\n",
" :param gain_setting: (FLOAT) Gain setting\n",
" :param photon_energy: (FLOAT) Photon Energy\n",
" :param only_dark: (BOOL) only retrieve dark constants\n",
" :param nodb_with_dark: (BOOL) no constant retrieval even for dark\n",
" :param cal_db_interface: (STR) the database interface port\n",
" :param creation_time: (STR) raw data creation time\n",
" :param corr_bools: (DICT) A dictionary with bools for applying requested corrections\n",
" :param pc_bools: (LIST) list of bools to retrieve pulse capacitor constants\n",
" :param inp: (LIST) input for the parallel cluster of the partial function\n",
" :return:\n",
" mdata_dict: (DICT) dictionary with the metadata for the retrieved constants\n",
" dev.device_name: (STR) device name\n",
" \"\"\"\n",
"\n",
" import numpy as np\n",
" import sys\n",
" import traceback\n",
" \n",
" from cal_tools.agipdlib import get_num_cells, get_acq_rate\n",
" from cal_tools.agipdutils import assemble_constant_dict\n",
" from cal_tools.tools import get_from_db\n",
"\n",
" from iCalibrationDB import Constants, Conditions, Detectors\n",
"\n",
" err = None\n",
"\n",
" qm_files, qm, dev, idx = inp\n",
" # get number of memory cells from a sequence file with image data\n",
" for f in qm_files:\n",
" if not max_cells:\n",
" max_cells = get_num_cells(f, karabo_id, idx)\n",
" if max_cells is None:\n",
" if f != qm_files[-1]:\n",
" continue\n",
" else:\n",
" raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
" else:\n",
" cells = np.arange(max_cells)\n",
" # get out of the loop,\n",
" # if max_cells is successfully calculated. \n",
" break\n",
"\n",
" if acq_rate == 0.:\n",
" acq_rate = get_acq_rate((f, karabo_id, idx))\n",
"\n",
" # avoid creating retireving constant, if requested.\n",
" if not nodb_with_dark: \n",
" const_dict = assemble_constant_dict(corr_bools, pc_bools, max_cells, bias_voltage,\n",
" gain_setting, acq_rate, photon_energy,\n",
" beam_energy=None, only_dark=only_dark)\n",
"\n",
" # Retrieve multiple constants through an input dictionary\n",
" # to return a dict of useful metadata.\n",
" mdata_dict = dict()\n",
" for cname, cval in const_dict.items():\n",
" try:\n",
" condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
" co, mdata = \\\n",
" get_from_db(dev, getattr(Constants.AGIPD, cname)(),\n",
" condition, getattr(np, cval[0])(cval[1]),\n",
" cal_db_interface, creation_time, meta_only=True, verbosity=0)\n",
" mdata_const = mdata.calibration_constant_version\n",
" # saving metadata in a dict\n",
" mdata_dict[cname] = dict()\n",
" # check if constant was sucessfully retrieved.\n",
" if mdata.comm_db_success: \n",
" mdata_dict[cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
" f\"{mdata_const.filename}\"\n",
" mdata_dict[cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
" else:\n",
" mdata_dict[cname][\"file-path\"] = const_dict[cname][:2]\n",
" mdata_dict[cname][\"creation-time\"] = None\n",
" except Exception as e:\n",
" err = f\"Error: {e}, Traceback: {traceback.format_exc()}\"\n",
" print(err)\n",
"\n",
" return qm, mdata_dict, dev.device_name, acq_rate, max_cells, err\n",
"\n",
"pc_bools = [corr_bools.get(\"rel_gain\"),\n",
" corr_bools.get(\"adjust_mg_baseline\"),\n",
" corr_bools.get('blc_noise'),\n",
" corr_bools.get('blc_hmatch'),\n",
" corr_bools.get('blc_stripes'),\n",
" melt_snow]\n",
"\n",
"inp = []\n",
"only_dark = False\n",
"nodb_with_dark = False\n",
"if not nodb:\n",
" only_dark=(calfile != \"\")\n",
"if calfile != \"\" and not corr_bools[\"only_offset\"]:\n",
" nodb_with_dark = nodb\n",
"\n",
"# A dict to connect virtual device\n",
"# to actual device name.\n",
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
" qm = f\"Q{i//4+1}M{i%4+1}\"\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" device = getattr(getattr(Detectors, dinstance), qm)\n",
" qm_files = [str(mapped_files[qm].get()) for _ in range(mapped_files[qm].qsize())]\n",
"\n",
" else:\n",
" print(f\"Skipping {qm}\")\n",
" continue\n",
"\n",
" inp.append((qm_files, qm, device, i))\n",
"\n",
"p = partial(retrieve_constants, karabo_id, bias_voltage, max_cells, \n",
" acq_rate, gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
" cal_db_interface, creation_time, \n",
" corr_bools, pc_bools)\n",
"\n",
"results = view.map_sync(p, inp)\n",
"#results = list(map(p, inp))\n",
"mod_dev = dict()\n",
"mdata_dict = dict()\n",
"for r in results:\n",
" if r:\n",
" qm, md_dict, dname, acq_rate, max_cells, err = r\n",
" mod_dev[dname] = {\"mod\": qm, \"err\": err}\n",
" if err:\n",
" print(f\"Error for module {qm}: {err}\")\n",
" mdata_dict[dname] = md_dict\n",
"# check if it is requested not to retrieve any constants from the database\n",
"if not nodb_with_dark:\n",
" with open(f\"{out_folder}/retrieved_constants.yml\", \"w\") as outfile:\n",
" yaml.safe_dump(mdata_dict, outfile)\n",
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
" print(\"\\nRetrieved constants for modules: \",\n",
" f\"{[', '.join([f'Q{x//4+1}M{x%4+1}' for x in modules])]}\")\n",
" print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {max_cells}\\n\"\n",
" f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Photon Energy: {photon_energy}\\n\")\n",
" print(f\"Constant metadata is saved in retrieved_constants.yml\\n\")\n",
"else:\n",
" print(\"No constants were retrieved as calibrated files will be used.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"Constants are retrieved with creation time: \")\n",
"i = 0\n",
"when = dict()\n",
"to_store = []\n",
"\n",
"for dname, dinfo in mod_dev.items():\n",
" print(dinfo[\"mod\"], \":\")\n",
" line = [dinfo[\"mod\"]]\n",
" if dname in mdata_dict:\n",
" for cname, mdata in mdata_dict[dname].items():\n",
" if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
" mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
" print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
" # Store few time stamps if exists\n",
" # Add NA to keep array structure\n",
" for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
" if not dname in mdata_dict or dinfo[\"err\"]:\n",
" line.append('Err')\n",
" else:\n",
" if cname in mdata_dict[dname]:\n",
" if mdata_dict[dname][cname][\"creation-time\"]:\n",
" line.append(mdata_dict[dname][cname][\"creation-time\"])\n",
" else:\n",
" line.append('NA')\n",
" else:\n",
" line.append('NA')\n",
" to_store.append(line)\n",
"\n",
" i += 1\n",
" if sequences:\n",
" seq_num = sequences[0]\n",
" else:\n",
" # if sequences[0] changed to None as it was -1\n",
" seq_num = 0\n",
" \n",
"with open(f\"{out_folder}/retrieved_constants.yml\",\"r\") as fyml:\n",
" time_summary = yaml.safe_load(fyml)\n",
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
" time_summary.update({\"time-summary\": {\n",
" \"SAll\":to_store\n",
" }})\n",
"with open(f\"{out_folder}/retrieved_constants.yml\",\"w\") as fyml:\n",
" yaml.safe_dump(time_summary, fyml)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
}
},
"nbformat": 4,
"nbformat_minor": 2
}