{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Combine Constants #\n",
"\n",
"Author: S. Hauf, Version: 0.1\n",
"\n",
"This notebook combines constants from various evaluations\n",
"\n",
"* dark image analysis, yielding offset and noise\n",
"* flat field analysis, yielding X-ray gain\n",
"* pulse capacitor analysis, yielding medium gain stage slopes and thresholds\n",
"* charge injection analysis, yielding low gain stage slopes and thresholds\n",
"\n",
"into a single set of calibration constants. These constants do not include offset and noise as they need to be reevaluated more frequently.\n",
"\n",
"Additionally, a bad pixel mask for all gain stages is deduced from the input. The mask contains dedicated entries for all pixels and memory cells as well as all three gains stages. "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# the following lines should be adjusted depending on data\n",
"modules = [-1] # modules to work on, range allowed\n",
"out_folder = \"/gpfs/exfel/exp/SPB/201830/p900019/usr/calibration0618/Merged1MHz2T\" # path to output to, required\n",
"local_output = True # output constants locally\n",
"db_output = False # output constants to database\n",
"bias_voltage = 300 # detector bias voltage\n",
"cal_db_interface = \"tcp://max-exfl016:5005\" # the database interface to use\n",
"mem_cells = 64 # number of memory cells used\n",
"instrument = \"SPB\"\n",
"photon_energy = 9.2 # the photon energy in keV\n",
"offset_store = \"/gpfs/exfel/exp/SPB/201830/p900019/usr/calibration0618/dark0.5MHz/agipd_offset_store_r0852_r0853_r0854.h5\" # for file-based access\n",
"gain_store = \"/gpfs/exfel/exp/SPB/201830/p900019/usr/calibration0618/FF/agipd_gain_store_r0820_modules_CHANID.h5\" # for file-based access\n",
"pc_store_base = \"/gpfs/exfel/exp/SPB/201831/p900039/usr/AGIPD/PC/Veto_1MHZ/agipd_pc_store_38_39_40_41_42_43_44_60_CHANID_CHANID.h5\" # for file-based access, use CHANID to indicate module ranges\n",
"ci_store = \"/gpfs/exfel/data/scratch/haufs/gain_Data/AGIPD_ci_data_74.h5\" # for file based access\n",
"high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. Runtime: ~ 1h\n",
"db_input = False # retreive data from calibration database, setting offset-store will overwrite this\n",
"deviation_threshold = 0.75 # peaks with an absolute location deviation larger than the medium are are considere bad\n",
"max_dev_high_gain = 0.2\n",
"max_dev_med_gain = 0.5\n",
"threshold_bounds_high_med = [100, 8100]\n",
"thresholds_offset_sigma = 5.\n",
"thresholds_offset_hard = [4000, 9000]\n",
"thresholds_noise_sigma = 10.\n",
"thresholds_noise_hard = [1, 20]\n",
"thresholds_xraygain_hard_high = [0.75, 1.25]\n",
"thresholds_xraygain_hard_med = [0.001, 0.25]\n",
"thresholds_xraygain_hard_low = [0.0001, 0.025]\n",
"no_flat_fields = False"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Each mask entry is encoded in 32 bits as:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [],
"source": [
"import tabulate\n",
"from cal_tools.enums import BadPixels\n",
"from IPython.display import HTML, Latex, Markdown, display\n",
"\n",
"table = []\n",
"for item in BadPixels:\n",
" table.append((item.name, \"{:016b}\".format(item.value)))\n",
"md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=[\"Bad pixel type\", \"Bit mask\"])))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Corrected data will include mask entries for each pixel and the evaluated gain stage:\n",
"\n",
" data[gmask != 0] = np.nan\n",
" \n",
"This will set all pixels with a mask entry in their gain stage to `np.nan`.\n",
"\n",
"Note that in the above table only the 16 least significant bits are shown. Higher bits are currently unused."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import os\n",
"\n",
"# std library imports\n",
"from functools import partial\n",
"\n",
"import h5py\n",
"import matplotlib\n",
"\n",
"# numpy and matplot lib specific\n",
"import numpy as np\n",
"\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.pyplot as plt\n",
"\n",
"%matplotlib inline\n",
"\n",
"import warnings\n",
"\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"from datetime import datetime\n",
"\n",
"import XFELDetAna.xfelpyanatools as xana\n",
"\n",
"# pyDetLib imports\n",
"import XFELDetAna.xfelpycaltools as xcal\n",
"from cal_tools.plotting import plot_badpix_3d, show_overview\n",
"from cal_tools.tools import (\n",
" gain_map_files,\n",
" get_notebook_name,\n",
" parse_runs,\n",
" run_prop_seq_from_path,\n",
")\n",
"from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
"\n",
"# usually no need to change these lines\n",
"sensor_size = [128, 512]\n",
"block_size = [64, 64]\n",
"QUADRANTS = 4\n",
"MODULES_PER_QUAD = 4\n",
"DET_FILE_INSET = \"AGIPD\"\n",
"\n",
"# the following lines should be adjusted depending on data\n",
"\n",
"\n",
"gain_output = \"{}/agipd_ff_store.h5\".format(out_folder)\n",
"\n",
"\n",
"\n",
"g_runs = [s for s in gain_store.split(\"_\") if s[0] == \"r\"]\n",
"n_gain_stores = 16\n",
"gain_store = gain_store.replace(\"CHANID\", \"{}\")\n",
"\n",
"n_pc_stores = 16\n",
"pc_store_base = pc_store_base.replace(\"CHANID\", \"{}\")\n",
"\n",
"\n",
"thresholds_xraygain_hard = (thresholds_xraygain_hard_high,\n",
" thresholds_xraygain_hard_med,\n",
" thresholds_xraygain_hard_low)\n",
"\n",
"# cells in raw data\n",
"max_cells = mem_cells\n",
"# actual memory cells: max_cells//2 if AGIPD is in interleaved mode\n",
"memory_cells = mem_cells\n",
"\n",
"# modules to characterize\n",
"if modules[0] == -1:\n",
" modules = range(16)\n",
"\n",
"NO_FLAT_FIELD_MODE = no_flat_fields\n",
"\n",
"\n",
"\n",
"# these lines can usually stay as is\n",
"gains = np.arange(3)\n",
"cells = np.arange(max_cells)\n",
"\n",
"pc_inj_time = None\n",
"ff_inj_time = None\n",
"ci_inj_time = None"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from collections import OrderedDict\n",
"\n",
"from dateutil import parser\n",
"\n",
"offset_g = OrderedDict()\n",
"noise_g = OrderedDict()\n",
"threshold_o = OrderedDict()\n",
"badpix_og = OrderedDict()\n",
"if not db_input or offset_store != \"\":\n",
" print(\"Offset, noise and thresholds have been read in from: {}\".format(offset_store))\n",
" store_file = h5py.File(offset_store, \"r\")\n",
" for i in modules:\n",
" qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
" offset_g[qm] = np.array(store_file[\"{}/Offset/0/data\".format(qm)])\n",
" noise_g[qm] = np.array(store_file[\"{}/Noise/0/data\".format(qm)])\n",
" threshold_o[qm] = np.array(store_file[\"{}/Threshold/0/data\".format(qm)])\n",
" badpix_og[qm] = np.array(store_file[\"{}/BadPixels/0/data\".format(qm)])\n",
" store_file.close()\n",
"else:\n",
" print(\"Offset, noise and thresholds have been read in from calibration database:\")\n",
" first_ex = True\n",
" for i in modules:\n",
" qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
" metadata = ConstantMetaData()\n",
" offset = Constants.AGIPD.Offset()\n",
" metadata.calibration_constant = offset\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.AGIPD(memory_cells=max_cells, bias_voltage=bias_voltage)\n",
" metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" metadata.calibration_constant_version = Versions.Now(device=getattr(Detectors.AGIPD1M1, qm))\n",
" metadata.retrieve(cal_db_interface)\n",
" offset_g[qm] = offset.data\n",
" \n",
" if first_ex:\n",
" print(\"Offset map was injected on: {}\".format(metadata.calibration_constant_version.begin_at))\n",
" \n",
" metadata = ConstantMetaData()\n",
" noise = Constants.AGIPD.Noise()\n",
" metadata.calibration_constant = noise\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.AGIPD(memory_cells=max_cells, bias_voltage=bias_voltage)\n",
" metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" metadata.calibration_constant_version = Versions.Now(device=getattr(Detectors.AGIPD1M1, qm))\n",
" metadata.retrieve(cal_db_interface)\n",
" noise_g[qm] = noise.data\n",
" \n",
" if first_ex:\n",
" print(\"Noise map was injected on: {}\".format(metadata.calibration_constant_version.begin_at))\n",
" \n",
" metadata = ConstantMetaData()\n",
" thresholds = Constants.AGIPD.ThresholdsDark()\n",
" metadata.calibration_constant = thresholds\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.AGIPD(memory_cells=max_cells, bias_voltage=bias_voltage)\n",
" metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" metadata.calibration_constant_version = Versions.Now(device=getattr(Detectors.AGIPD1M1, qm))\n",
" metadata.retrieve(cal_db_interface)\n",
" threshold_o[qm] = thresholds.data\n",
" \n",
" if first_ex:\n",
" print(\"Threshold map was injected on: {}\".format(metadata.calibration_constant_version.begin_at))\n",
" first_ex = False\n",
" \n",
" metadata = ConstantMetaData()\n",
" bpix = Constants.AGIPD.BadPixelsDark()\n",
" metadata.calibration_constant = bpix\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.AGIPD(memory_cells=max_cells, bias_voltage=bias_voltage)\n",
" metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" metadata.calibration_constant_version = Versions.Now(device=getattr(Detectors.AGIPD1M1, qm))\n",
" metadata.retrieve(cal_db_interface)\n",
" badpix_og[qm] = bpix.data\n",
" \n",
" if first_ex:\n",
" print(\"Bad pixel map was injected on: {}\".format(metadata.calibration_constant_version.begin_at))\n",
" first_ex = False"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"xray_gain_m = OrderedDict()\n",
"xray_gain_b = OrderedDict()\n",
"xray_mask_gain = OrderedDict()\n",
"xray_entries = OrderedDict()\n",
"medians = []\n",
"if not NO_FLAT_FIELD_MODE:\n",
" \n",
" if not db_input or gain_store != \"\":\n",
" \n",
" print(\"Reading gain data from {} files:\".format(n_gain_stores))\n",
" for j in range(n_gain_stores):\n",
" store_file = h5py.File(gain_store.format(j), \"r\")\n",
" qm = \"Q{}M{}\".format(j//4+1, j%4+1)\n",
" print(\"{}: {}\".format(qm, store_file))\n",
" xgm = store_file[\"/{}/Gain/0/data\".format(qm)][()]\n",
" \n",
" medians.append(np.nanmedian(xgm))\n",
" else:\n",
" print(\"Reading gain data from calibration database:\")\n",
" first_ex = True\n",
" for j in modules:\n",
" qm = \"Q{}M{}\".format(j//4+1, j%4+1)\n",
" device = getattr(Detectors.AGIPD1M1, qm)\n",
" # gains related\n",
" metadata = ConstantMetaData()\n",
" gain = Constants.AGIPD.SlopesFF()\n",
" metadata.calibration_constant = gain\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Illuminated.AGIPD(memory_cells, bias_voltage, 9.2,\n",
" pixels_x=512, pixels_y=128, beam_energy=None)\n",
"\n",
" metadata.detector_condition = condition\n",
"\n",
" metadata.calibration_constant_version = Versions.Now(device=device)\n",
" metadata.retrieve(cal_db_interface)\n",
" \n",
" if first_ex:\n",
" print(\"FF gain map was injected on: {}\".format(metadata.calibration_constant_version.begin_at))\n",
" first_ex = False\n",
" \n",
" medians.append(np.nanmedian(gain.data[...,0]))\n",
" \n",
" \n",
" global_med = np.nanmedian(np.array(medians))\n",
" \n",
" \n",
" if local_output:\n",
" store_file_out = h5py.File(gain_output.format(\"_\".join(g_runs)), \"w\")\n",
" print(\"Outputting to intermediate file {}:\".format(store_file_out))\n",
" \n",
" \n",
" k = 0\n",
"\n",
" for j in (modules if db_input else range(n_gain_stores)):\n",
"\n",
" qm = \"Q{}M{}\".format(j//4+1, j%4+1)\n",
" qmout = qm\n",
"\n",
" xgm = np.zeros(sensor_size+[memory_cells], np.float32)\n",
" xgb = np.zeros(sensor_size+[memory_cells], np.float32)\n",
" xge = np.zeros(sensor_size+[memory_cells, 5], np.float32)\n",
" xgk = np.zeros(sensor_size+[memory_cells], np.float32)\n",
"\n",
" if not db_input or gain_store != \"\":\n",
" if ff_inj_time is None:\n",
" ff_inj_time = os.stat(gain_store.format(j)).st_mtime\n",
" store_file = h5py.File(gain_store.format(j), \"r\")\n",
"\n",
" xgm = store_file[\"/{}/Gain/0/data\".format(qm)][()]/global_med\n",
" xgb = store_file[\"/{}/GainOffset/0/data\".format(qm)][()]\n",
" xge = store_file[\"/{}/Entries/0/data\".format(qm)][()]\n",
" xgk = store_file[\"/{}/BadPixels/0/data\".format(qm)][()]\n",
" store_file.close()\n",
" \n",
" else:\n",
" device = getattr(Detectors.AGIPD1M1, qm)\n",
" # gains related\n",
" metadata = ConstantMetaData()\n",
" gain = Constants.AGIPD.SlopesFF()\n",
" metadata.calibration_constant = gain\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Illuminated.AGIPD(memory_cells, bias_voltage, 9.2,\n",
" pixels_x=512, pixels_y=128, beam_energy=None)\n",
"\n",
" metadata.detector_condition = condition\n",
"\n",
" metadata.calibration_constant_version = Versions.Now(device=device)\n",
" metadata.retrieve(cal_db_interface)\n",
" \n",
" xgm = gain.data[...,0]\n",
" xgb = gain.data[...,1]\n",
" xge = None\n",
" \n",
" metadata = ConstantMetaData()\n",
" gainbp = Constants.AGIPD.BadPixelsFF()\n",
" metadata.calibration_constant = gain\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Illuminated.AGIPD(memory_cells, bias_voltage, 9.2,\n",
" pixels_x=512, pixels_y=128, beam_energy=None)\n",
"\n",
" metadata.detector_condition = condition\n",
"\n",
" metadata.calibration_constant_version = Versions.Now(device=device)\n",
" metadata.retrieve(cal_db_interface)\n",
" xgk = gainbp.data\n",
"\n",
" if local_output:\n",
" store_file_out[\"/{}/Gain/0/data\".format(qmout)] = xgm\n",
" store_file_out[\"/{}/GainOffset/0/data\".format(qmout)] = xgb\n",
" store_file_out[\"/{}/BadPixels/0/data\".format(qmout)] = xgk\n",
"\n",
" xray_gain_m[qmout] = xgm\n",
" xray_gain_b[qmout] = xgb\n",
" xray_entries[qmout] = xge\n",
" xray_mask_gain[qmout] = xgk\n",
" k += 1\n",
" \n",
" if local_output:\n",
" store_file_out.close()\n",
"else:\n",
" for i in range(16):\n",
" qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
" xray_gain_m[qm] = np.ones(sensor_size + [memory_cells], np.float32)\n",
" xray_gain_b[qm] = np.zeros(sensor_size + [memory_cells], np.float32)\n",
" xray_entries[qm] = np.ones(sensor_size + [memory_cells], np.float32)\n",
" xray_mask_gain[qm] = np.ones(sensor_size + [memory_cells], np.uint8)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"scrolled": true
},
"outputs": [],
"source": [
"pc_high_m = OrderedDict()\n",
"pc_high_b = OrderedDict()\n",
"pc_med_m = OrderedDict()\n",
"pc_med_b = OrderedDict()\n",
"pc_med_o = OrderedDict()\n",
"pc_med_c = OrderedDict()\n",
"pc_med_a = OrderedDict()\n",
"pc_thresh = OrderedDict()\n",
"pc_thresh_bounds = OrderedDict()\n",
"pc_mask = OrderedDict()\n",
"pc_high_dev = OrderedDict()\n",
"pc_med_dev = OrderedDict()\n",
"\n",
"\n",
"if not db_input or pc_store_base != \"\":\n",
" \n",
" for i in range(n_pc_stores):\n",
" ofile = pc_store_base.format(i, i)\n",
" \n",
" if pc_inj_time is None:\n",
" pc_inj_time = os.stat(ofile).st_mtime\n",
" \n",
" store_file = h5py.File(ofile, \"r\")\n",
"\n",
" qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
" qmout = qm\n",
" pc_high_m[qmout] = np.moveaxis(store_file[\"/{}/{}/0/data\".format(qm, 'ml')][()], 0, 2)[...,:memory_cells]\n",
" pc_high_b[qmout] = np.moveaxis(store_file[\"/{}/{}/0/data\".format(qm, 'bl')][()], 0, 2)[...,:memory_cells]\n",
" pc_high_dev[qmout] = np.moveaxis(store_file[\"/{}/{}/0/data\".format(qm, 'devl')][()], 0, 2)[...,:memory_cells]\n",
" pc_med_dev[qmout] = np.moveaxis(store_file[\"/{}/{}/0/data\".format(qm, 'devh')][()], 0, 2)[...,:memory_cells]\n",
" pc_med_m[qmout] = np.moveaxis(store_file[\"/{}/{}/0/data\".format(qm, 'mh')][()], 0, 2)[...,:memory_cells]\n",
" pc_med_b[qmout] = np.moveaxis(store_file[\"/{}/{}/0/data\".format(qm, 'bh')][()], 0, 2)[...,:memory_cells]\n",
"\n",
" pc_med_o[qmout] = np.moveaxis(store_file[\"/{}/{}/0/data\".format(qm, 'oh')][()], 0, 2)[...,:memory_cells]\n",
" pc_med_c[qmout] = np.moveaxis(store_file[\"/{}/{}/0/data\".format(qm, 'ch')][()], 0, 2)[...,:memory_cells]\n",
" pc_med_a[qmout] = np.moveaxis(store_file[\"/{}/{}/0/data\".format(qm, 'ah')][()], 0, 2)[...,:memory_cells]\n",
" pc_thresh[qmout] = np.moveaxis(store_file[\"/{}/{}/0/data\".format(qm, 'tresh')][()], 0, 2)[...,:memory_cells]\n",
" pc_thresh_bounds[qmout] = np.moveaxis(store_file[\"/{}/{}/0/data\".format(qm, 'tresh_bounds')][()], 0, 2)[...,:memory_cells]\n",
" mask = np.zeros(pc_high_m[qmout].shape, np.uint8)\n",
" mask[(pc_thresh[qmout] < threshold_bounds_high_med[0]) | (pc_thresh[qmout] > threshold_bounds_high_med[1])] += 1\n",
" mask[(pc_high_dev[qmout] == 0) | (pc_med_dev[qmout] == 0)] += 2\n",
" mask[(pc_high_dev[qmout] > max_dev_high_gain) | (pc_med_dev[qmout] >= max_dev_med_gain)] += 4\n",
" mask[(pc_high_dev[qmout] < 0) | (pc_med_dev[qmout] < 0)] += 8\n",
" mask[(~np.isfinite(pc_high_dev[qmout])) | (~np.isfinite(pc_med_dev[qmout]))] += 16\n",
" pc_mask[qmout] = mask\n",
" store_file.close()\n",
"else:\n",
" print(\"Reading PC data from calibration database:\")\n",
" first_ex = True\n",
" for i in modules:\n",
" qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
" qmout = qm\n",
" device = getattr(Detectors.AGIPD1M1, qm)\n",
" metadata = ConstantMetaData()\n",
" pcdata = Constants.AGIPD.SlopesPC()\n",
" metadata.calibration_constant = pcdata\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.AGIPD(memory_cells=74, bias_voltage=bias_voltage)\n",
"\n",
" metadata.detector_condition = condition\n",
"\n",
" metadata.calibration_constant_version = Versions.Now(device=device)\n",
" metadata.retrieve(cal_db_interface)\n",
" \n",
" pc_high_m[qmout] = np.moveaxis(pcdata.data[0,...], 0, 2)[...,:memory_cells]\n",
" pc_high_b[qmout] = np.moveaxis(pcdata.data[1,...], 0, 2)[...,:memory_cells]\n",
" pc_high_dev[qmout] = np.moveaxis(pcdata.data[2,...], 0, 2)[...,:memory_cells]\n",
" pc_med_dev[qmout] = np.moveaxis(pcdata.data[8,...], 0, 2)[...,:memory_cells]\n",
" pc_med_m[qmout] = np.moveaxis(pcdata.data[3,...], 0, 2)[...,:memory_cells]\n",
" pc_med_b[qmout] = np.moveaxis(pcdata.data[4,...], 0, 2)[...,:memory_cells]\n",
"\n",
" pc_med_o[qmout] = np.moveaxis(pcdata.data[5,...], 0, 2)[...,:memory_cells]\n",
" pc_med_c[qmout] = np.moveaxis(pcdata.data[6,...], 0, 2)[...,:memory_cells]\n",
" pc_med_a[qmout] = np.moveaxis(pcdata.data[7,...], 0, 2)[...,:memory_cells]\n",
" pc_thresh[qmout] = np.moveaxis(pcdata.data[9,...], 0, 2)[...,:memory_cells]\n",
" #pc_thresh_bounds[qmout] = np.moveaxis(store_file[\"/{}/{}/0/data\".format(qm, 'tresh_bounds')][()], 0, 2)[...,:memory_cells]\n",
" pc_thresh_bounds[qmout] = None\n",
" mask = np.zeros(pc_high_m[qmout].shape, np.uint8)\n",
" mask[(pc_thresh[qmout] < threshold_bounds_high_med[0]) | (pc_thresh[qmout] > threshold_bounds_high_med[1])] += 1\n",
" mask[(pc_high_dev[qmout] == 0) | (pc_med_dev[qmout] == 0)] += 2\n",
" mask[(pc_high_dev[qmout] > max_dev_high_gain) | (pc_med_dev[qmout] >= max_dev_med_gain)] += 4\n",
" mask[(pc_high_dev[qmout] < 0) | (pc_med_dev[qmout] < 0)] += 8\n",
" mask[(~np.isfinite(pc_high_dev[qmout])) | (~np.isfinite(pc_med_dev[qmout]))] += 16\n",
" pc_mask[qmout] = mask\n",
" \n",
" if first_ex:\n",
" print(\"PC gain map was injected on: {}\".format(metadata.calibration_constant_version.begin_at))\n",
" first_ex = False\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"store_file = h5py.File(ci_store, \"r\")\n",
"ci_gain_correlation = OrderedDict()\n",
"ci_gain_b = OrderedDict()\n",
"ci_threshold = OrderedDict()\n",
"ci_mask = OrderedDict()\n",
"\n",
"if ci_inj_time is None:\n",
" ci_inj_time = os.stat(ci_store).st_mtime\n",
"\n",
"for i in range(16):\n",
" qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
" \n",
" ci_gain_correlation[qm] = store_file[\"/{}/SlopeCorrelation/0/data\".format(qm)][()][...,:memory_cells]\n",
" ci_gain_b[qm] = store_file[\"/{}/BaseOffset/0/data\".format(qm)][()][...,:memory_cells,:]\n",
" ci_threshold[qm] = store_file[\"/{}/Threshold/0/data\".format(qm)][()][...,:memory_cells,:]\n",
" ci_mask[qm] = store_file[\"/{}/SlopeCorrelation/0/mask\".format(qm)][()][...,:memory_cells,:]\n",
" \n",
"store_file.close()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"constants = OrderedDict()\n",
"\n",
"for i in range(16): \n",
" qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
" constants[qm] = {}\n",
" g_m = np.zeros((sensor_size[0], sensor_size[1], memory_cells, 3), np.float32)\n",
" g_b = np.zeros((sensor_size[0], sensor_size[1], memory_cells, 3), np.float32)\n",
" g_o = np.zeros((sensor_size[0], sensor_size[1], memory_cells), np.float32)\n",
" g_c = np.zeros((sensor_size[0], sensor_size[1], memory_cells), np.float32)\n",
" g_a = np.zeros((sensor_size[0], sensor_size[1], memory_cells), np.float32)\n",
" fac_high_med = pc_med_m[qm]/pc_high_m[qm]\n",
" pc_gain_rel = pc_med_m[qm]*(np.nanmedian(xray_gain_m[qm])/np.nanmedian(pc_med_m[qm]))\n",
" g_m[...,0] = xray_gain_m[qm]\n",
" g_m[...,1] = xray_gain_m[qm]*fac_high_med\n",
" g_m[...,2] = g_m[...,1]*ci_gain_correlation[qm]\n",
" g_b[...,0] = xray_gain_b[qm]\n",
" g_b[...,1] = pc_med_b[qm] - offset_g[qm][...,1] + xray_gain_b[qm] \n",
" g_b[...,2] = ci_gain_b[qm][...,2] - offset_g[qm][...,2] + xray_gain_b[qm]\n",
" \n",
" g_o = pc_med_o[qm] - (xray_gain_b[qm]-pc_high_b[qm])/(pc_high_m[qm]-xray_gain_m[qm])\n",
" g_a = pc_med_a[qm]\n",
" g_c = pc_med_c[qm] * xray_gain_m[qm]/pc_high_m[qm]\n",
" \n",
" constants[qm]['RelativeGain'] = g_m\n",
" constants[qm]['BaseOffset'] = g_b\n",
" constants[qm]['RelativeGainNonLinOffset'] = g_o\n",
" constants[qm]['RelativeGainNonLinAmplitude'] = g_a - xray_gain_b[qm]\n",
" constants[qm]['RelativeGainNonLinScale'] = g_c\n",
" \n",
" th = np.zeros((sensor_size[0], sensor_size[1], memory_cells, 2), np.float32)\n",
" th[...,0] = pc_thresh[qm][...]#threshold_o[qm][...,0]\n",
" th[...,1] = ci_threshold[qm][...,1]\n",
" constants[qm]['Threshold'] = th\n",
" constants[qm]['ThresholdBounds'] = pc_thresh_bounds[qm]\n",
" \n",
" # bad pixels\n",
" bp = np.zeros((sensor_size[0], sensor_size[1], memory_cells, 3), np.uint32)\n",
" # offset related bad pixels\n",
" offset = offset_g[qm]\n",
" offset_mn = np.nanmedian(offset, axis=(0,1))\n",
" offset_std = np.nanstd(offset, axis=(0,1)) \n",
" \n",
" bp |= badpix_og[qm]\n",
" \n",
" # noise related bad pixels\n",
" noise = noise_g[qm]\n",
" noise_mn = np.nanmedian(noise, axis=(0,1))\n",
" noise_std = np.nanstd(noise, axis=(0,1)) \n",
" \n",
" # bad pixels from X-ray gain data\n",
" xm = xray_mask_gain[qm]\n",
" for i in range(3):\n",
" #tbp = bp[...,i]\n",
" #tbp[xm != 0] |= 2**1\n",
" #tbp[(g_m[...,i] < thresholds_xraygain_hard[i][0]) |\n",
" # (g_m[...,i] > thresholds_xraygain_hard[i][1])] |= 2**1\n",
" #if i > 0:\n",
" # tbp[(tbp[...,0] != 0) & (tbp[...,i] == 0)] |= 2**1\n",
" \n",
" bp[...,i] |= xm\n",
" \n",
" # bad pixels for PC data\n",
" pcm = pc_mask[qm]\n",
" bp[...,0] |= pcm\n",
" bp[...,1] |= pcm\n",
" \n",
" # bad pixels for ci data\n",
" tbp = bp[...,2]\n",
" tbp[ci_mask != 0] |= BadPixels.CI2_EVAL_ERROR.value\n",
" bp[...,2] = tbp\n",
" \n",
" # threshold related bad pixels\n",
" \n",
" bp[~np.isfinite(th[...,0]), 0] |= BadPixels.GAIN_THRESHOLDING_ERROR.value\n",
" bp[th[...,0] == 0, 0] |= BadPixels.GAIN_THRESHOLDING_ERROR.value\n",
" \n",
" bp[~np.isfinite(th[...,0]), 1] |= BadPixels.GAIN_THRESHOLDING_ERROR.value\n",
" bp[th[...,0] == 0, 1] |= BadPixels.GAIN_THRESHOLDING_ERROR.value\n",
" \n",
" bp[~np.isfinite(th[...,1]), 1] |= BadPixels.GAIN_THRESHOLDING_ERROR.value\n",
" bp[th[...,1] == 0, 1] |= BadPixels.GAIN_THRESHOLDING_ERROR.value\n",
" \n",
" bp[~np.isfinite(th[...,1]), 2] |= BadPixels.GAIN_THRESHOLDING_ERROR.value\n",
" bp[th[...,1] == 0, 2] |= BadPixels.GAIN_THRESHOLDING_ERROR.value\n",
" \n",
" constants[qm]['BadPixels'] = bp\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"ofile = \"{}/agipd_base_store_{}_{}.h5\".format(out_folder, memory_cells, \"_\".join(g_runs))\n",
"store_file = h5py.File(ofile, \"w\")\n",
"for qm, r in constants.items(): \n",
" for key, item in r.items():\n",
" \n",
" store_file[\"/{}/{}/0/data\".format(qm, key)] = item \n",
"store_file.close()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
"from mpl_toolkits.axes_grid1 import AxesGrid\n",
"\n",
"\n",
"def show_overview(cell_to_preview, gain_to_preview):\n",
" \n",
" for module, data in constants.items():\n",
" fig = plt.figure(figsize=(20,20))\n",
" grid = AxesGrid(fig, 111,\n",
" nrows_ncols=(4, 2),\n",
" axes_pad=(0.9, 0.15),\n",
" label_mode=\"1\",\n",
" share_all=True,\n",
" cbar_location=\"right\",\n",
" cbar_mode=\"each\",\n",
" cbar_size=\"7%\",\n",
" cbar_pad=\"2%\",\n",
" )\n",
" i = 0\n",
" for key, item in data.items():\n",
" try:\n",
" cf = 0\n",
" if \"Threshold\" in key:\n",
" cf = -1\n",
" if len(item.shape) == 4:\n",
" med = np.nanmedian(item[...,cell_to_preview, gain_to_preview + cf])\n",
" else:\n",
" med = np.nanmedian(item[...,cell_to_preview])\n",
"\n",
" bound = 0.2\n",
" while(np.count_nonzero((item[...,cell_to_preview, gain_to_preview + cf] < med-np.abs(bound*med)) |\n",
" (item[...,cell_to_preview, gain_to_preview + cf] > med+np.abs(bound*med)))/item[...,cell_to_preview, gain_to_preview + cf].size > 0.01): \n",
" bound *=2\n",
"\n",
" if \"BadPixels\" in key:\n",
" im = grid[i].imshow(np.log2(item[...,cell_to_preview, gain_to_preview + cf]).astype(np.float32), interpolation=\"nearest\",\n",
" vmin=0, vmax=8, aspect='auto')\n",
" pass\n",
" \n",
" else:\n",
" \n",
" if len(item.shape) == 4:\n",
" im = grid[i].imshow(item[...,cell_to_preview, gain_to_preview + cf], interpolation=\"nearest\",\n",
" vmin=med-np.abs(bound*med), vmax=np.abs(med+bound*med), aspect='auto')\n",
" else:\n",
" im = grid[i].imshow(item[...,cell_to_preview], interpolation=\"nearest\",\n",
" vmin=med-np.abs(bound*med), vmax=med+np.abs(bound*med), aspect='auto')\n",
" cb = grid.cbar_axes[i].colorbar(im)\n",
"\n",
" grid[i].text(5, 20, key, color=\"w\" if key != \"BadPixels\" else \"k\", fontsize=20)\n",
"\n",
" i += 1\n",
" except:\n",
" pass\n",
" \n",
" grid[0].text(5, 50, module, color=\"r\" if key != \"BadPixels\" else \"k\", fontsize=20)\n",
" #grid[0].text(5, 20, module, color=\"r\" if key != \"BadPixels\" else \"k\", fontsize=20)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Overview of Modules ##\n",
"\n",
"The following plots give an overview of relevant constants for each gain for a single memory cell: 4. The constants are as following:\n",
"\n",
"* RelativeGain: the per-pixel, per-gain, per-memory-cell relative gain assuming a linear gain function $G(x)$.\n",
"\n",
"* Threshold: the gain switching threshold, will only be shown accurately for in the medium and low gain plots. Here the threshold is between high and medium gain and medium and low gain respectively.\n",
"\n",
"* ThresholdBounds: the bound value at which the upper threshold of a gain stage was determined.\n",
"\n",
"* Badpixels: the bad pixel map for the gain stage\n",
"\n",
"* BaseOffset: the gain-dependent offset which needs to be taken into account in addition to the dark image derived offset.\n",
"\n",
"Additionally, the results from fitting the pulse capacitor data for the transition region are given.\n",
"\n",
"* RelativeGainNonLinOffset: the non-linear offset component determined from PC data of the transition region. Usually note needed for correction, as linearity can sufficiently be assumed.\n",
"\n",
"* RelativeGainNonLinScale: the non-linear scale component determined from PC data of the transition region. Usually note needed for correction, as linearity can sufficiently be assumed.\n",
"\n",
"* RelativeGainNonLinAmplitude: the non-linear amplitude component determined from PC data of the transition region. Usually note needed for correction, as linearity can sufficiently be assumed.\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### High Gain ###"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"show_overview(4, 0)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Medium Gain ###"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"scrolled": true
},
"outputs": [],
"source": [
"show_overview(12, 1)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Low Gain ###"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"show_overview(4, 2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"if db_output:\n",
" # inject data read in with injection time set to date of the data files\n",
" # we only do this for non-offset data\n",
" pc_iso = datetime.fromtimestamp(pc_inj_time)\n",
" ff_iso = datetime.fromtimestamp(ff_inj_time)\n",
" ci_iso = datetime.fromtimestamp(ci_inj_time)\n",
" print(\"Using {} as injection time for PC data\".format(pc_iso))\n",
" print(\"Using {} as injection time for FF data\".format(ff_iso))\n",
" print(\"Using {} as injection time for CI data\".format(ci_iso))\n",
" for i in range(16):\n",
" qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
" metadata = ConstantMetaData()\n",
" slopespc = Constants.AGIPD.SlopesPC()\n",
" \n",
" data = np.zeros(list(pc_high_m[qm].shape) + [10], np.float32)\n",
" data[...,0] = pc_high_m[qm]\n",
" data[...,1] = pc_high_b[qm]\n",
" data[...,2] = pc_high_dev[qm]\n",
" data[...,3] = pc_med_dev[qm]\n",
" data[...,4] = pc_med_m[qm]\n",
" data[...,5] = pc_med_b[qm]\n",
" data[...,6] = pc_med_o[qm]\n",
" data[...,7] = pc_med_c[qm]\n",
" data[...,8] = pc_med_a[qm]\n",
" data[...,9] = pc_thresh[qm]\n",
" \n",
" slopespc.data = data\n",
" metadata.calibration_constant = slopespc\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.AGIPD(memory_cells=mem_cells, bias_voltage=bias_voltage)\n",
" metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" metadata.calibration_constant_version = Versions.Timespan(device=getattr(Detectors.AGIPD1M1, qm), start=pc_iso)\n",
" metadata.send(cal_db_interface)\n",
" \n",
" metadata = ConstantMetaData()\n",
" slopesff = Constants.AGIPD.SlopesFF()\n",
" \n",
" slopesff.data = xray_gain_m[qm]\n",
" metadata.calibration_constant = slopesff\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Illuminated.AGIPD(memory_cells, bias_voltage, photon_energy,\n",
" pixels_x=512, pixels_y=128, beam_energy=None)\n",
" metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" metadata.calibration_constant_version = Versions.Now(device=getattr(Detectors.AGIPD1M1, qm))#Timespan(device=getattr(Detectors.AGIPD1M1, qm), start=ff_iso)\n",
" metadata.send(cal_db_interface)\n",
" \n",
" metadata = ConstantMetaData()\n",
" slopesci = Constants.AGIPD.SlopesCI()\n",
" \n",
"\n",
" slopesci.data = ci_gain_correlation[qm]\n",
" metadata.calibration_constant = slopesci\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.AGIPD(memory_cells=mem_cells, bias_voltage=bias_voltage)\n",
" metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" metadata.calibration_constant_version = Versions.Timespan(device=getattr(Detectors.AGIPD1M1, qm), start=ci_iso)\n",
" metadata.send(cal_db_interface)\n",
" \n",
" \n",
" \n",
" metadata = ConstantMetaData()\n",
" thresholdspc = Constants.AGIPD.ThresholdsPC()\n",
" \n",
"\n",
" thresholdspc.data = pc_thresh[qm][...]#threshold_o[qm][...,0]\n",
" \n",
" metadata.calibration_constant = thresholdspc\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.AGIPD(memory_cells=mem_cells, bias_voltage=bias_voltage)\n",
" metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" metadata.calibration_constant_version = Versions.Timespan(device=getattr(Detectors.AGIPD1M1, qm), start=pc_iso)\n",
" metadata.send(cal_db_interface)\n",
" \n",
" \n",
" metadata = ConstantMetaData()\n",
" thresholdsci = Constants.AGIPD.ThresholdsCI()\n",
" \n",
"\n",
" thresholdsci.data = ci_threshold[qm][...,1]\n",
" metadata.calibration_constant = thresholdsci\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.AGIPD(memory_cells=mem_cells, bias_voltage=bias_voltage)\n",
" metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" metadata.calibration_constant_version = Versions.Timespan(device=getattr(Detectors.AGIPD1M1, qm), start=ci_iso)\n",
" metadata.send(cal_db_interface)\n",
"\n",
" "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Global Bad Pixel Behaviour ##\n",
"\n",
"The following plots show the results of bad pixel evaluation for all evaluated memory cells. Cells are stacked in the Z-dimension, while pixels values in x/y are rebinned with a factor of 4. This excludes single bad pixels present only in disconnected pixels. Hence, any bad pixels spanning at least 4 pixels in the x/y-plane, or across at least two memory cells are indicated. Colors encode the bad pixel type, or mixed type."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### High Gain ###"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"cols = {1: ('Bad pixel', '#FF000080')}\n",
"\n",
"rebin = 4 if not high_res_badpix_3d else 2\n",
"\n",
"gain = 0\n",
"for mod, data in constants.items():\n",
" bp = data['BadPixels'][...,gain]\n",
" any_bd = np.zeros_like(bp)\n",
" any_bd[bp != 0] = 1\n",
" plot_badpix_3d(any_bd, cols, title=mod, rebin_fac=rebin, azim=22.5)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Medium Gain ###"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"gain = 1\n",
"for mod, data in constants.items():\n",
" bp = data['BadPixels'][...,gain]\n",
" any_bd = np.zeros_like(bp)\n",
" any_bd[bp != 0] = 1\n",
" plot_badpix_3d(any_bd, cols, title=mod, rebin_fac=rebin, azim=22.5)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Low Gain ###\n",
"\n",
"We currently skip output here, as all cells and pixels have not fully accurate data."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"#gain = 2\n",
"#for mod, data in constants.items():\n",
"# bp = data['BadPixels'][...,gain]\n",
"# any_bd = np.zeros_like(bp)\n",
"# any_bd[bp != 0] = 1\n",
"# plot_badpix_3d(any_bd, cols, title=mod, rebin_fac=rebin, azim=22.5)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.6"
}
},
"nbformat": 4,
"nbformat_minor": 2
}