{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# AGIPD Retrieving Constants Pre-correction #\n",
"\n",
"Author: K.Ahmed, Version: 1.0\n",
"\n",
"Retrieving Required Constants for Offline Calibration of the AGIPD Detector"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-02-21T11:30:06.730220Z",
"start_time": "2019-02-21T11:30:06.658286Z"
}
},
"outputs": [],
"source": [
"cluster_profile = \"noDB\"\n",
"in_folder = \"/gpfs/exfel/exp/SPB/202030/p900119/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_\" # the folder to output to, required\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
"run = 80 # runs to process, required\n",
"\n",
"karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
"path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
"h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP_TEST' # path to control information\n",
"karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
"karabo_da_control = 'AGIPD1MCTRL00' # karabo DA for control infromation\n",
"\n",
"use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
"creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
"\n",
"calfile = \"\" # path to calibration file. Leave empty if all data should come from DB\n",
"nodb = False # if set only file-based constants will be used\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
"bias_voltage = 300\n",
"acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
"gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
"photon_energy = 9.2 # photon energy in keV\n",
"max_cells_db_dark = 0 # set to a value different than 0 to use this value for dark data DB queries\n",
"max_cells_db = 0 # set to a value different than 0 to use this value for DB queries\n",
"\n",
"# Correction Booleans\n",
"only_offset = False # Apply only Offset correction. if False, Offset is applied by Default. if True, Offset is only applied.\n",
"rel_gain = False # do relative gain correction based on PC data\n",
"xray_gain = True # do relative gain correction based on xray data\n",
"blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
"blc_stripes = False # if set, baseline corrected via stripes\n",
"blc_hmatch = False # if set, base line correction via histogram matching is attempted \n",
"match_asics = False # if set, inner ASIC borders are matched to the same signal level\n",
"adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Fill dictionaries comprising bools and arguments for correction and data analysis\n",
"\n",
"# Here the herarichy and dependability for correction booleans are defined \n",
"corr_bools = {}\n",
"\n",
"# offset is at the bottom of AGIPD correction pyramid.\n",
"corr_bools[\"only_offset\"] = only_offset\n",
"\n",
"# Dont apply any corrections if only_offset is requested \n",
"if not only_offset:\n",
" corr_bools[\"adjust_mg_baseline\"] = adjust_mg_baseline\n",
" corr_bools[\"rel_gain\"] = rel_gain\n",
" corr_bools[\"xray_corr\"] = xray_gain\n",
" corr_bools[\"blc_noise\"] = blc_noise\n",
" corr_bools[\"blc_hmatch\"] = blc_hmatch"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"from collections import OrderedDict\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
"import os\n",
"import h5py\n",
"import numpy as np\n",
"import matplotlib\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.pyplot as plt\n",
"from ipyparallel import Client\n",
"print(f\"Connecting to profile {cluster_profile}\")\n",
"view = Client(profile=cluster_profile)[:]\n",
"view.use_dill()\n",
"\n",
"from iCalibrationDB import Constants, Conditions, Detectors\n",
"from cal_tools.tools import (map_modules_from_folder, get_dir_creation_date)\n",
"from cal_tools.agipdlib import get_gain_setting\n",
"from dateutil import parser\n",
"from datetime import timedelta"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-02-21T11:30:07.086286Z",
"start_time": "2019-02-21T11:30:06.929722Z"
}
},
"outputs": [],
"source": [
"max_cells = mem_cells\n",
"\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run)\n",
" offset = parser.parse(creation_date_offset)\n",
" delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
" creation_time += delta\n",
" print(f\"Using {creation_time} as creation time\")\n",
"\n",
"if sequences[0] == -1:\n",
" sequences = None\n",
"\n",
"if in_folder[-1] == \"/\":\n",
" in_folder = in_folder[:-1]\n",
"print(f\"Outputting to {out_folder}\")\n",
"\n",
"os.makedirs(out_folder, exist_ok=True)\n",
"\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"from cal_tools.agipdlib import SnowResolution\n",
"\n",
"melt_snow = False if corr_bools[\"only_offset\"] else SnowResolution.NONE"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"control_fname = f'{in_folder}/r{run:04d}/RAW-R{run:04d}-{karabo_da_control}-S00000.h5'\n",
"h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
"\n",
"if gain_setting == 0.1:\n",
" if creation_time.replace(tzinfo=None) < parser.parse('2020-01-31'):\n",
" print(\"Set gain-setting to None for runs taken before 2020-01-31\")\n",
" gain_setting = None\n",
" else:\n",
" try:\n",
" gain_setting = get_gain_setting(control_fname, h5path_ctrl)\n",
" except Exception as e:\n",
" print(f'Error while reading gain setting: {e}\\n')\n",
" \n",
"print(f\"Gain setting: {gain_setting}\")\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"\n",
"if karabo_da[0] == '-1':\n",
" if modules[0] == -1:\n",
" modules = list(range(16))\n",
" karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-02-21T11:30:07.974174Z",
"start_time": "2019-02-21T11:30:07.914832Z"
}
},
"outputs": [],
"source": [
"# set everything up filewise\n",
"print(f\"Checking the files before retrieving constants\")\n",
"mmf = map_modules_from_folder(in_folder, run, path_template, karabo_da, sequences)\n",
"\n",
"mapped_files, mod_ids, total_sequences, sequences_qm, _ = mmf"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Retrieve Constants ##"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from functools import partial\n",
"import yaml\n",
"\n",
"def retrieve_constants(karabo_id, bias_voltage, max_cells, acq_rate, \n",
" gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
" cal_db_interface, creation_time, \n",
" corr_bools, pc_bools, inp):\n",
" \"\"\"\n",
" Retreive constant for each module in parallel and produce a dictionary\n",
" with the creation-time and constant file path.\n",
" \n",
" :param karabo_id: (STR) Karabo ID\n",
" :param bias_voltage: (FLOAT) Bias Voltage\n",
" :param max_cells: (INT) Memory cells\n",
" :param acq_rate: (FLOAT) Acquisition Rate\n",
" :param gain_setting: (FLOAT) Gain setting\n",
" :param photon_energy: (FLOAT) Photon Energy\n",
" :param only_dark: (BOOL) only retrieve dark constants\n",
" :param nodb_with_dark: (BOOL) no constant retrieval even for dark\n",
" :param cal_db_interface: (STR) the database interface port\n",
" :param creation_time: (STR) raw data creation time\n",
" :param corr_bools: (DICT) A dictionary with bools for applying requested corrections\n",
" :param pc_bools: (LIST) list of bools to retrieve pulse capacitor constants\n",
" :param inp: (LIST) input for the parallel cluster of the partial function\n",
" :return:\n",
" mdata_dict: (DICT) dictionary with the metadata for the retrieved constants\n",
" dev.device_name: (STR) device name\n",
" \"\"\"\n",
"\n",
" import numpy as np\n",
" import sys\n",
" import traceback\n",
" \n",
" from cal_tools.agipdlib import get_num_cells, get_acq_rate\n",
" from cal_tools.agipdutils import assemble_constant_dict\n",
" from cal_tools.tools import get_from_db\n",
"\n",
" from iCalibrationDB import Constants, Conditions, Detectors\n",
"\n",
" err = None\n",
"\n",
" qm_files, qm, dev, idx = inp\n",
" # get number of memory cells from a sequence file with image data\n",
" for f in qm_files:\n",
" if max_cells == 0:\n",
" max_cells = get_num_cells(f, karabo_id, idx)\n",
" if max_cells is None:\n",
" if f != qm_files[-1]:\n",
" continue\n",
" else:\n",
" raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
" else:\n",
" cells = np.arange(max_cells)\n",
" # get out of the loop,\n",
" # if max_cells is successfully calculated. \n",
" break\n",
"\n",
" if acq_rate == 0.:\n",
" acq_rate = get_acq_rate((f, karabo_id, idx))\n",
" else:\n",
" acq_rate = None\n",
"\n",
" print(f\"Set memory cells to {max_cells}\")\n",
" print(f\"Set acquistion rate cells to {acq_rate} MHz\")\n",
"\n",
" # avoid creating retireving constant, if requested.\n",
" if not nodb_with_dark: \n",
" const_dict = assemble_constant_dict(corr_bools, pc_bools, max_cells, bias_voltage,\n",
" gain_setting, acq_rate, photon_energy,\n",
" beam_energy=None, only_dark=only_dark)\n",
"\n",
" # Retrieve multiple constants through an input dictionary\n",
" # to return a dict of useful metadata.\n",
" mdata_dict = dict()\n",
" for cname, cval in const_dict.items():\n",
" try:\n",
" condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
" co, mdata = \\\n",
" get_from_db(dev, getattr(Constants.AGIPD, cname)(),\n",
" condition, getattr(np, cval[0])(cval[1]),\n",
" cal_db_interface, creation_time, meta_only=True)\n",
" mdata_const = mdata.calibration_constant_version\n",
" # saving metadata in a dict\n",
" mdata_dict[cname] = dict()\n",
" # check if constant was sucessfully retrieved.\n",
" if mdata.comm_db_success: \n",
" mdata_dict[cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
" f\"{mdata_const.filename}\"\n",
" mdata_dict[cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
" else:\n",
" mdata_dict[cname][\"file-path\"] = const_dict[cname]\n",
" mdata_dict[cname][\"creation-time\"] = None\n",
" except Exception as e:\n",
" err = f\"Error: {e}, Traceback: {traceback.format_exc()}\"\n",
" print(err)\n",
"\n",
" return qm, mdata_dict, dev.device_name, acq_rate, max_cells, err\n",
"\n",
"pc_bools = [corr_bools.get(\"rel_gain\"),\n",
" corr_bools.get(\"adjust_mg_baseline\"),\n",
" corr_bools.get('blc_noise'),\n",
" corr_bools.get('blc_hmatch'),\n",
" corr_bools.get('blc_stripes'),\n",
" melt_snow]\n",
"\n",
"# Extracting Instrument string\n",
"instrument = karabo_id.split(\"_\")[0]\n",
"if instrument == \"SPB\":\n",
" dinstance = \"AGIPD1M1\"\n",
"else:\n",
" dinstance = \"AGIPD1M2\"\n",
"\n",
"print(f\"Instrument {instrument}\")\n",
"print(f\"Detector instance {dinstance}\")\n",
"\n",
"inp = []\n",
"only_dark = False\n",
"nodb_with_dark = False\n",
"if not nodb:\n",
" only_dark=(calfile != \"\")\n",
"if calfile != \"\" and not corr_bools[\"only_offset\"]:\n",
" nodb_with_dark = nodb\n",
"\n",
"# A dict to connect virtual device\n",
"# to actual device name.\n",
"for i in modules:\n",
" qm = f\"Q{i//4+1}M{i%4+1}\"\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" device = getattr(getattr(Detectors, dinstance), qm)\n",
" qm_files = [str(mapped_files[qm].get()) for _ in range(mapped_files[qm].qsize())]\n",
"\n",
" else:\n",
" print(f\"Skipping {qm}\")\n",
" continue\n",
"\n",
" inp.append((qm_files, qm, device, i))\n",
"\n",
"p = partial(retrieve_constants, karabo_id, bias_voltage, max_cells, \n",
" acq_rate, gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
" cal_db_interface, creation_time, \n",
" corr_bools, pc_bools)\n",
"\n",
"results = view.map_sync(p, inp)\n",
"#results = list(map(p, inp))\n",
"mod_dev = dict()\n",
"mdata_dict = dict()\n",
"for r in results:\n",
" if r:\n",
" qm, md_dict, dname, acq_rate, max_cells, err = r\n",
" mod_dev[dname] = {\"mod\": qm, \"err\": err}\n",
" if err:\n",
" print(f\"Error for module {qm}: {err}\")\n",
" mdata_dict[dname] = md_dict\n",
"# check if it is requested not to retrieve any constants from the database\n",
"if not nodb_with_dark:\n",
" with open(f\"{out_folder}/retrieved_constants.yml\", \"w\") as outfile:\n",
" yaml.dump(mdata_dict, outfile)\n",
" print(\"\\nRetrieved constants for modules: \",\n",
" f\"{[', '.join([f'Q{x//4+1}M{x%4+1}' for x in modules])]}\")\n",
" print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {max_cells}\\n\"\n",
" f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Photon Energy: {photon_energy}\\n\")\n",
" print(f\"Constant metadata is saved in retrieved_constants.yml\\n\")\n",
"else:\n",
" print(\"No constants were retrieved as calibrated files will be used.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"Constants are retrieved with creation time: \")\n",
"i = 0\n",
"when = dict()\n",
"to_store = []\n",
"\n",
"for dname, dinfo in mod_dev.items():\n",
" print(dinfo[\"mod\"], \":\")\n",
" line = [dinfo[\"mod\"]]\n",
" if dname in mdata_dict:\n",
" for cname, mdata in mdata_dict[dname].items():\n",
" if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
" mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
" print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
" # Store few time stamps if exists\n",
" # Add NA to keep array structure\n",
" for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
" if not dname in mdata_dict or dinfo[\"err\"]:\n",
" line.append('Err')\n",
" else:\n",
" if cname in mdata_dict[dname]:\n",
" if mdata_dict[dname][cname][\"creation-time\"]:\n",
" line.append(mdata_dict[dname][cname][\"creation-time\"])\n",
" else:\n",
" line.append('NA')\n",
" else:\n",
" line.append('NA')\n",
" to_store.append(line)\n",
"\n",
" i += 1\n",
" if sequences:\n",
" seq_num = sequences[0]\n",
" else:\n",
" # if sequences[0] changed to None as it was -1\n",
" seq_num = 0\n",
" \n",
"with open(f\"{out_folder}/retrieved_constants.yml\",\"r\") as fyml:\n",
" time_summary = yaml.load(fyml)\n",
" time_summary.update({\"time-summary\": {\n",
" \"SAll\":to_store\n",
" }})\n",
"with open(f\"{out_folder}/retrieved_constants.yml\",\"w\") as fyml:\n",
" yaml.safe_dump(time_summary, fyml)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
}
},
"nbformat": 4,
"nbformat_minor": 2
}