{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# AGIPD Retrieving Constants Pre-correction #\n",
"\n",
"Author: European XFEL Detector Group, Version: 1.0\n",
"\n",
"Retrieving Required Constants for Offline Calibration of the AGIPD Detector"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-02-21T11:30:06.730220Z",
"start_time": "2019-02-21T11:30:06.658286Z"
}
},
"outputs": [],
"source": [
"cluster_profile = \"noDB\"\n",
"in_folder = \"/gpfs/exfel/exp/SPB/202030/p900119/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_\" # the folder to output to, required\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
"run = 80 # runs to process, required\n",
"\n",
"karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
"path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
"h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP_TEST' # path to control information\n",
"karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
"karabo_da_control = 'AGIPD1MCTRL00' # karabo DA for control infromation\n",
"\n",
"use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
"creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
"\n",
"slopes_ff_from_files = \"\" # Path to locally stored SlopesFF and BadPixelsFF constants\n",
"calfile = \"\" # path to calibration file. Leave empty if all data should come from DB\n",
"nodb = False # if set only file-based constants will be used\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
"bias_voltage = 300\n",
"acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
"gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
"photon_energy = 9.2 # photon energy in keV\n",
"max_cells_db_dark = 0 # set to a value different than 0 to use this value for dark data DB queries\n",
"max_cells_db = 0 # set to a value different than 0 to use this value for DB queries\n",
"\n",
"# Correction Booleans\n",
"only_offset = False # Apply only Offset correction. if False, Offset is applied by Default. if True, Offset is only applied.\n",
"rel_gain = False # do relative gain correction based on PC data\n",
"xray_gain = True # do relative gain correction based on xray data\n",
"blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
"blc_stripes = False # if set, baseline corrected via stripes\n",
"blc_hmatch = False # if set, base line correction via histogram matching is attempted \n",
"match_asics = False # if set, inner ASIC borders are matched to the same signal level\n",
"adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Fill dictionaries comprising bools and arguments for correction and data analysis\n",
"\n",
"# Here the herarichy and dependability for correction booleans are defined \n",
"corr_bools = {}\n",
"\n",
"# offset is at the bottom of AGIPD correction pyramid.\n",
"corr_bools[\"only_offset\"] = only_offset\n",
"\n",
"# Dont apply any corrections if only_offset is requested \n",
"if not only_offset:\n",
" corr_bools[\"adjust_mg_baseline\"] = adjust_mg_baseline\n",
" corr_bools[\"rel_gain\"] = rel_gain\n",
" corr_bools[\"xray_corr\"] = xray_gain\n",
" corr_bools[\"blc_noise\"] = blc_noise\n",
" corr_bools[\"blc_hmatch\"] = blc_hmatch"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"from collections import OrderedDict\n",
"\n",
"import os\n",
"import h5py\n",
"import numpy as np\n",
"import matplotlib\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.pyplot as plt\n",
"import multiprocessing as mp\n",
"\n",
"from iCalibrationDB import Constants, Conditions, Detectors\n",
"from cal_tools.tools import (map_modules_from_folder, get_dir_creation_date)\n",
"from cal_tools.agipdlib import get_gain_setting\n",
"from dateutil import parser\n",
"from datetime import timedelta"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-02-21T11:30:07.086286Z",
"start_time": "2019-02-21T11:30:06.929722Z"
}
},
"outputs": [],
"source": [
"max_cells = mem_cells\n",
"\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run)\n",
" offset = parser.parse(creation_date_offset)\n",
" delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
" creation_time += delta\n",
" print(f\"Using {creation_time} as creation time\")\n",
"\n",
"if sequences[0] == -1:\n",
" sequences = None\n",
"\n",
"if in_folder[-1] == \"/\":\n",
" in_folder = in_folder[:-1]\n",
"print(f\"Outputting to {out_folder}\")\n",
"\n",
"os.makedirs(out_folder, exist_ok=True)\n",
"\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"from cal_tools.agipdlib import SnowResolution\n",
"\n",
"melt_snow = False if corr_bools[\"only_offset\"] else SnowResolution.NONE"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"control_fname = f'{in_folder}/r{run:04d}/RAW-R{run:04d}-{karabo_da_control}-S00000.h5'\n",
"h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
"\n",
"if gain_setting == 0.1:\n",
" if creation_time.replace(tzinfo=None) < parser.parse('2020-01-31'):\n",
" print(\"Set gain-setting to None for runs taken before 2020-01-31\")\n",
" gain_setting = None\n",
" else:\n",
" try:\n",
" gain_setting = get_gain_setting(control_fname, h5path_ctrl)\n",
" except Exception as e:\n",
" print(f'ERROR: while reading gain setting from: \\n{control_fname}')\n",
" print(e)\n",
" print(\"Set gain setting to 0\")\n",
" gain_setting = 0\n",
"\n",
"print(f\"Gain setting: {gain_setting}\")\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"\n",
"\n",
"# Extracting Instrument string\n",
"instrument = karabo_id.split(\"_\")[0]\n",
"# Evaluate detector instance for mapping\n",
"if instrument == \"SPB\":\n",
" dinstance = \"AGIPD1M1\"\n",
" nmods = 16\n",
"elif instrument == \"MID\":\n",
" dinstance = \"AGIPD1M2\"\n",
" nmods = 16\n",
"elif instrument == \"HED\":\n",
" dinstance = \"AGIPD500K\"\n",
" nmods = 8\n",
"\n",
"print(f\"Instrument {instrument}\")\n",
"print(f\"Detector instance {dinstance}\")\n",
"\n",
"\n",
"if karabo_da[0] == '-1':\n",
" if modules[0] == -1:\n",
" modules = list(range(nmods))\n",
" karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-02-21T11:30:07.974174Z",
"start_time": "2019-02-21T11:30:07.914832Z"
}
},
"outputs": [],
"source": [
"# set everything up filewise\n",
"print(f\"Checking the files before retrieving constants\")\n",
"mmf = map_modules_from_folder(in_folder, run, path_template, karabo_da, sequences)\n",
"\n",
"mapped_files, mod_ids, total_sequences, sequences_qm, _ = mmf"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Retrieve Constants ##"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from functools import partial\n",
"import yaml\n",
"\n",
"\n",
"def retrieve_constants(karabo_id, bias_voltage, max_cells, acq_rate, \n",
" gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
" cal_db_interface, creation_time, \n",
" corr_bools, pc_bools, inp):\n",
" \"\"\"\n",
" Retreive constant for each module in parallel and produce a dictionary\n",
" with the creation-time and constant file path.\n",
" \n",
" :param karabo_id: (STR) Karabo ID\n",
" :param bias_voltage: (FLOAT) Bias Voltage\n",
" :param max_cells: (INT) Memory cells\n",
" :param acq_rate: (FLOAT) Acquisition Rate\n",
" :param gain_setting: (FLOAT) Gain setting\n",
" :param photon_energy: (FLOAT) Photon Energy\n",
" :param only_dark: (BOOL) only retrieve dark constants\n",
" :param nodb_with_dark: (BOOL) no constant retrieval even for dark\n",
" :param cal_db_interface: (STR) the database interface port\n",
" :param creation_time: (STR) raw data creation time\n",
" :param corr_bools: (DICT) A dictionary with bools for applying requested corrections\n",
" :param pc_bools: (LIST) list of bools to retrieve pulse capacitor constants\n",
" :param inp: (LIST) input for the parallel cluster of the partial function\n",
" :return:\n",
" mdata_dict: (DICT) dictionary with the metadata for the retrieved constants\n",
" dev.device_name: (STR) device name\n",
" \"\"\"\n",
"\n",
" import numpy as np\n",
" import sys\n",
" import traceback\n",
" \n",
" from cal_tools.agipdlib import get_num_cells, get_acq_rate\n",
" from cal_tools.agipdutils import assemble_constant_dict\n",
" from cal_tools.tools import get_from_db\n",
"\n",
" from iCalibrationDB import Constants, Conditions, Detectors\n",
"\n",
" err = None\n",
"\n",
" qm_files, qm, dev, idx = inp\n",
" # get number of memory cells from a sequence file with image data\n",
" for f in qm_files:\n",
" if not max_cells:\n",
" max_cells = get_num_cells(f, karabo_id, idx)\n",
" if max_cells is None:\n",
" if f != qm_files[-1]:\n",
" continue\n",
" else:\n",
" raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
" else:\n",
" cells = np.arange(max_cells)\n",
" # get out of the loop,\n",
" # if max_cells is successfully calculated. \n",
" break\n",
"\n",
" if acq_rate == 0.:\n",
" acq_rate = get_acq_rate((f, karabo_id, idx))\n",
"\n",
" # avoid creating retireving constant, if requested.\n",
" if not nodb_with_dark:\n",
" const_dict = assemble_constant_dict(corr_bools, pc_bools, max_cells, bias_voltage,\n",
" gain_setting, acq_rate, photon_energy,\n",
" beam_energy=None, only_dark=only_dark)\n",
"\n",
" # Retrieve multiple constants through an input dictionary\n",
" # to return a dict of useful metadata.\n",
" mdata_dict = dict()\n",
" for cname, cval in const_dict.items():\n",
" # saving metadata in a dict\n",
" mdata_dict[cname] = dict()\n",
" \n",
" if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
" mdata_dict[cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
" mdata_dict[cname][\"creation-time\"] = \"00:00:00\"\n",
" else:\n",
" try:\n",
" condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
" co, mdata = \\\n",
" get_from_db(dev, getattr(Constants.AGIPD, cname)(),\n",
" condition, getattr(np, cval[0])(cval[1]),\n",
" cal_db_interface, creation_time, meta_only=True, verbosity=0)\n",
" mdata_const = mdata.calibration_constant_version\n",
" \n",
" # check if constant was sucessfully retrieved.\n",
" if mdata.comm_db_success: \n",
" mdata_dict[cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
" f\"{mdata_const.filename}\"\n",
" mdata_dict[cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
" else:\n",
" mdata_dict[cname][\"file-path\"] = const_dict[cname][:2]\n",
" mdata_dict[cname][\"creation-time\"] = None\n",
" except Exception as e:\n",
" err = f\"Error: {e}, Traceback: {traceback.format_exc()}\"\n",
" print(err)\n",
"\n",
" return qm, mdata_dict, dev.device_name, acq_rate, max_cells, err\n",
"\n",
"pc_bools = [corr_bools.get(\"rel_gain\"),\n",
" corr_bools.get(\"adjust_mg_baseline\"),\n",
" corr_bools.get('blc_noise'),\n",
" corr_bools.get('blc_hmatch'),\n",
" corr_bools.get('blc_stripes'),\n",
" melt_snow]\n",
"\n",
"inp = []\n",
"only_dark = False\n",
"nodb_with_dark = False\n",
"if not nodb:\n",
" only_dark=(calfile != \"\")\n",
"if calfile != \"\" and not corr_bools[\"only_offset\"]:\n",
" nodb_with_dark = nodb\n",
"\n",
"# A dict to connect virtual device\n",
"# to actual device name.\n",
"for i in modules:\n",
" qm = f\"Q{i//4+1}M{i%4+1}\"\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" device = getattr(getattr(Detectors, dinstance), qm)\n",
" qm_files = [str(mapped_files[qm].get()) for _ in range(mapped_files[qm].qsize())]\n",
"\n",
" else:\n",
" print(f\"Skipping {qm}\")\n",
" continue\n",
"\n",
" inp.append((qm_files, qm, device, i))\n",
"\n",
"p = partial(retrieve_constants, karabo_id, bias_voltage, max_cells, \n",
" acq_rate, gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
" cal_db_interface, creation_time, \n",
" corr_bools, pc_bools)\n",
"\n",
"with mp.Pool(processes=16) as pool:\n",
" results = pool.map(p, inp)\n",
"\n",
"mod_dev = dict()\n",
"mdata_dict = dict()\n",
"for r in results:\n",
" if r:\n",
" qm, md_dict, dname, acq_rate, max_cells, err = r\n",
" mod_dev[dname] = {\"mod\": qm, \"err\": err}\n",
" if err:\n",
" print(f\"Error for module {qm}: {err}\")\n",
" mdata_dict[dname] = md_dict\n",
"# check if it is requested not to retrieve any constants from the database\n",
"if not nodb_with_dark:\n",
" with open(f\"{out_folder}/retrieved_constants.yml\", \"w\") as outfile:\n",
" yaml.safe_dump(mdata_dict, outfile)\n",
" print(\"\\nRetrieved constants for modules: \",\n",
" f\"{[', '.join([f'Q{x//4+1}M{x%4+1}' for x in modules])]}\")\n",
" print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {max_cells}\\n\"\n",
" f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Photon Energy: {photon_energy}\\n\")\n",
" print(f\"Constant metadata is saved in retrieved_constants.yml\\n\")\n",
"else:\n",
" print(\"No constants were retrieved as calibrated files will be used.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"Constants are retrieved with creation time: \")\n",
"i = 0\n",
"when = dict()\n",
"to_store = []\n",
"\n",
"for dname, dinfo in mod_dev.items():\n",
" print(dinfo[\"mod\"], \":\")\n",
" line = [dinfo[\"mod\"]]\n",
" if dname in mdata_dict:\n",
" for cname, mdata in mdata_dict[dname].items():\n",
" if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
" mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
" print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
" # Store few time stamps if exists\n",
" # Add NA to keep array structure\n",
" for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
" if not dname in mdata_dict or dinfo[\"err\"]:\n",
" line.append('Err')\n",
" else:\n",
" if cname in mdata_dict[dname]:\n",
" if mdata_dict[dname][cname][\"creation-time\"]:\n",
" line.append(mdata_dict[dname][cname][\"creation-time\"])\n",
" else:\n",
" line.append('NA')\n",
" else:\n",
" line.append('NA')\n",
" to_store.append(line)\n",
"\n",
" i += 1\n",
" if sequences:\n",
" seq_num = sequences[0]\n",
" else:\n",
" # if sequences[0] changed to None as it was -1\n",
" seq_num = 0\n",
" \n",
"with open(f\"{out_folder}/retrieved_constants.yml\",\"r\") as fyml:\n",
" time_summary = yaml.safe_load(fyml)\n",
" time_summary.update({\"time-summary\": {\n",
" \"SAll\":to_store\n",
" }})\n",
"with open(f\"{out_folder}/retrieved_constants.yml\",\"w\") as fyml:\n",
" yaml.safe_dump(time_summary, fyml)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
}
},
"nbformat": 4,
"nbformat_minor": 2
}