diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 722eb0a113265f456e3044a81ff7ff7dd31c30d3..8aee1ac01698ad49dfa214d937779f92acb06c16 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -86,6 +86,7 @@
"import h5py\n",
"import matplotlib\n",
"import numpy as np\n",
+ "import pasha as psh\n",
"import tabulate\n",
"import yaml\n",
"\n",
@@ -95,6 +96,7 @@
"\n",
"%matplotlib inline\n",
"\n",
+ "import itertools\n",
"import multiprocessing\n",
"\n",
"from cal_tools.agipdlib import (\n",
@@ -104,9 +106,7 @@
" get_gain_setting,\n",
" get_num_cells,\n",
")\n",
- "\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
- "\n",
"from cal_tools.plotting import (\n",
" create_constant_overview,\n",
" plot_badpix_3d,\n",
@@ -125,7 +125,7 @@
" save_const_to_h5,\n",
" send_to_db,\n",
")\n",
- "from iCalibrationDB import Conditions, Constants, Detectors"
+ "import iCalibrationDB"
]
},
{
@@ -211,7 +211,7 @@
" gsettings.append(get_gain_setting(control_fname, h5path_ctrl))\n",
" if not all(g == gsettings[0] for g in gsettings):\n",
" raise ValueError(f\"Different gain settings for the 3 input runs {gsettings}\")\n",
- " gain_setting = gsettings[0]\n",
+ " gain_setting = gsettings[0]\n",
" except Exception as e:\n",
" print(f'Error while reading gain setting from: \\n{control_fname}')\n",
" print(f'Error: {e}')\n",
@@ -245,7 +245,7 @@
"print(\"Parameters are:\")\n",
"print(f\"Proposal: {prop}\")\n",
"print(f\"Memory cells: {mem_cells}/{max_cells}\")\n",
- "print(\"Runs: {}\".format([ v for v in offset_runs.values()]))\n",
+ "print(\"Runs: {}\".format([v for v in offset_runs.values()]))\n",
"print(f\"Sequences: {sequences}\")\n",
"print(f\"Interlaced mode: {interlaced}\")\n",
"print(f\"Using DB: {db_output}\")\n",
@@ -256,35 +256,6 @@
"print(f\"Operation mode is {'fixed' if fixed_gain_mode else 'adaptive'} gain mode\")"
]
},
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "The following lines will create a queue of files which will the be executed module-parallel. Distiguishing between different gains."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# set everything up filewise\n",
- "os.makedirs(out_folder, exist_ok=True)\n",
- "gmf = map_gain_stages(in_folder, offset_runs, path_template, karabo_da, sequences)\n",
- "gain_mapped_files, total_sequences, total_file_size = gmf\n",
- "print(f\"Will process a total of {total_sequences} files.\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Calculate Offsets, Noise and Thresholds ##\n",
- "\n",
- "The calculation is performed per-pixel and per-memory-cell. Offsets are simply the median value for a set of dark data taken at a given gain, noise the standard deviation, and gain-bit values the medians of the gain array."
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -306,7 +277,7 @@
" thresholds_offset_hard_mg,\n",
" thresholds_offset_hard_lg,\n",
" ]\n",
- "print(f\"Will use the following hard offset thresholds\")\n",
+ "print(\"Will use the following hard offset thresholds\")\n",
"for name, value in zip((\"High\", \"Medium\", \"Low\"), thresholds_offset_hard):\n",
" print(f\"- {name} gain: {value}\")\n",
"\n",
@@ -320,13 +291,64 @@
" ]"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The following lines will create a queue of files which will the be executed module-parallel. Distiguishing between different gains."
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
- "def characterize_module(fast_data_filename: str, channel: int, gg: int) -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
+ "# set everything up filewise\n",
+ "os.makedirs(out_folder, exist_ok=True)\n",
+ "gain_mapped_files, total_files, total_file_size = map_gain_stages(\n",
+ " in_folder, offset_runs, path_template, karabo_da, sequences\n",
+ ")\n",
+ "print(f\"Will process a total of {total_files} files ({total_file_size:.02f} GB).\")\n",
+ "\n",
+ "inp = []\n",
+ "for gain_index, (gain, qm_file_map) in enumerate(gain_mapped_files.items()):\n",
+ " for module_index in modules:\n",
+ " qm = module_index_to_qm(module_index)\n",
+ " if qm not in qm_file_map:\n",
+ " print(f\"Did not find files for {qm}\")\n",
+ " continue\n",
+ " file_queue = qm_file_map[qm]\n",
+ " while not file_queue.empty():\n",
+ " filename = file_queue.get()\n",
+ " print(f\"Process {filename} for {qm}\")\n",
+ " inp.append((filename, module_index, gain_index))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Calculate Offsets, Noise and Thresholds ##\n",
+ "\n",
+ "The calculation is performed per-pixel and per-memory-cell. Offsets are simply the median value for a set of dark data taken at a given gain, noise the standard deviation, and gain-bit values the medians of the gain array."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# min() only relevant if running on multiple modules (i.e. within notebook)\n",
+ "parallel_num_procs = min(12, total_files)\n",
+ "parallel_num_threads = multiprocessing.cpu_count() // parallel_num_procs\n",
+ "print(f\"Will use {parallel_num_procs} processes with {parallel_num_threads} threads each\")\n",
+ "\n",
+ "\n",
+ "def characterize_module(\n",
+ " fast_data_filename: str, channel: int, gain_index: int\n",
+ ") -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
" if max_cells == 0:\n",
" num_cells = get_num_cells(fast_data_filename, karabo_id, channel)\n",
" else:\n",
@@ -335,14 +357,14 @@
" print(f\"Using {num_cells} memory cells\")\n",
"\n",
" if acq_rate == 0.:\n",
- " slow_paths = control_names[gg], karabo_id_control\n",
+ " slow_paths = control_names[gain_index], karabo_id_control\n",
" fast_paths = fast_data_filename, karabo_id, channel\n",
" local_acq_rate = get_acq_rate(fast_paths, slow_paths)\n",
" else:\n",
" local_acq_rate = acq_rate\n",
"\n",
- " local_thresholds_offset_hard = thresholds_offset_hard[gg]\n",
- " local_thresholds_noise_hard = thresholds_noise_hard[gg]\n",
+ " local_thresholds_offset_hard = thresholds_offset_hard[gain_index]\n",
+ " local_thresholds_noise_hard = thresholds_noise_hard[gain_index]\n",
"\n",
" h5path_f = h5path.format(channel)\n",
" h5path_idx_f = h5path_idx.format(channel)\n",
@@ -362,64 +384,76 @@
" last_index = int(last[status != 0][-1]) + 1\n",
" first_index = int(first[status != 0][0])\n",
" im = np.array(infile[f\"{h5path_f}/data\"][first_index:last_index,...])\n",
- " cellIds = np.squeeze(infile[f\"{h5path_f}/cellId\"][first_index:last_index,...])\n",
- " \n",
+ " cell_ids = np.squeeze(infile[f\"{h5path_f}/cellId\"][first_index:last_index,...])\n",
+ "\n",
" if interlaced:\n",
" if not fixed_gain_mode:\n",
" ga = im[1::2, 0, ...]\n",
" im = im[0::2, 0, ...].astype(np.float32)\n",
- " cellIds = cellIds[::2]\n",
+ " cell_ids = cell_ids[::2]\n",
" else:\n",
" if not fixed_gain_mode:\n",
" ga = im[:, 1, ...]\n",
" im = im[:, 0, ...].astype(np.float32)\n",
"\n",
- " im = np.rollaxis(im, 2)\n",
- " im = np.rollaxis(im, 2, 1)\n",
- "\n",
+ " im = np.transpose(im)\n",
" if not fixed_gain_mode:\n",
- " ga = np.rollaxis(ga, 2)\n",
- " ga = np.rollaxis(ga, 2, 1)\n",
- " \n",
- " offset = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
- " noise = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ " ga = np.transpose(ga)\n",
+ "\n",
+ " context = psh.context.ThreadContext(num_workers=parallel_num_threads)\n",
+ " offset = context.alloc(shape=(im.shape[0], im.shape[1], num_cells), dtype=np.float64)\n",
+ " noise = context.alloc(like=offset)\n",
"\n",
" if fixed_gain_mode:\n",
" gains = None\n",
" gains_std = None\n",
" else:\n",
- " gains = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
- " gains_std = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
- "\n",
- " for cc in np.unique(cellIds[cellIds < num_cells]):\n",
- " cellidx = cellIds == cc\n",
- " offset[...,cc] = np.median(im[..., cellidx], axis=2)\n",
- " noise[...,cc] = np.std(im[..., cellidx], axis=2)\n",
+ " gains = context.alloc(like=offset)\n",
+ " gains_std = context.alloc(like=offset)\n",
+ "\n",
+ " def process_cell(worker_id, array_index, cell_number):\n",
+ " cell_slice_index = (cell_ids == cell_number)\n",
+ " im_slice = im[..., cell_slice_index]\n",
+ " offset[..., cell_number] = np.median(im_slice, axis=2)\n",
+ " noise[..., cell_number] = np.std(im_slice, axis=2)\n",
" if not fixed_gain_mode:\n",
- " gains[...,cc] = np.median(ga[..., cellidx], axis=2)\n",
- " gains_std[...,cc] = np.std(ga[..., cellidx], axis=2)\n",
+ " ga_slice = ga[..., cell_slice_index]\n",
+ " gains[..., cell_number] = np.median(ga_slice, axis=2)\n",
+ " gains_std[..., cell_number] = np.std(ga_slice, axis=2)\n",
+ "\n",
+ " context.map(process_cell, np.unique(cell_ids))\n",
"\n",
" # bad pixels\n",
- " bp = np.zeros(offset.shape, np.uint32)\n",
+ " bp = np.zeros_like(offset, dtype=np.uint32)\n",
" # offset related bad pixels\n",
" offset_mn = np.nanmedian(offset, axis=(0,1))\n",
" offset_std = np.nanstd(offset, axis=(0,1))\n",
"\n",
" bp[(offset < offset_mn-thresholds_offset_sigma*offset_std) |\n",
- " (offset > offset_mn+thresholds_offset_sigma*offset_std)] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
- " bp[(offset < local_thresholds_offset_hard[0]) | (\n",
- " offset > local_thresholds_offset_hard[1])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
- " bp[~np.isfinite(offset)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
+ " (offset > offset_mn+thresholds_offset_sigma*offset_std)] |= BadPixels.OFFSET_OUT_OF_THRESHOLD\n",
+ " bp[(offset < local_thresholds_offset_hard[0]) |\n",
+ " (offset > local_thresholds_offset_hard[1])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD\n",
+ " bp[~np.isfinite(offset)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR\n",
"\n",
" # noise related bad pixels\n",
" noise_mn = np.nanmedian(noise, axis=(0,1))\n",
" noise_std = np.nanstd(noise, axis=(0,1))\n",
" bp[(noise < noise_mn-thresholds_noise_sigma*noise_std) |\n",
- " (noise > noise_mn+thresholds_noise_sigma*noise_std)] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
- " bp[(noise < local_thresholds_noise_hard[0]) | (noise > local_thresholds_noise_hard[1])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
- " bp[~np.isfinite(noise)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
+ " (noise > noise_mn+thresholds_noise_sigma*noise_std)] |= BadPixels.NOISE_OUT_OF_THRESHOLD\n",
+ " bp[(noise < local_thresholds_noise_hard[0]) | (noise > local_thresholds_noise_hard[1])] |= BadPixels.NOISE_OUT_OF_THRESHOLD\n",
+ " bp[~np.isfinite(noise)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR\n",
"\n",
- " return offset, noise, gains, gains_std, gg, bp, num_cells, local_acq_rate"
+ " return offset, noise, gains, gains_std, bp, num_cells, local_acq_rate"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with multiprocessing.Pool(processes=parallel_num_procs) as pool:\n",
+ " results = pool.starmap(characterize_module, inp)"
]
},
{
@@ -434,49 +468,30 @@
"if not fixed_gain_mode:\n",
" gain_g = OrderedDict()\n",
" gainstd_g = OrderedDict()\n",
- " \n",
- "start = datetime.now()\n",
+ "\n",
"all_cells = []\n",
"all_acq_rate = []\n",
"\n",
- "inp = []\n",
- "for gg, (gain, mapped_files) in enumerate(gain_mapped_files.items()):\n",
- " dones = []\n",
- " for i in modules:\n",
- " qm = module_index_to_qm(i)\n",
- " if qm in mapped_files and not mapped_files[qm].empty():\n",
- " fname_in = mapped_files[qm].get()\n",
- " print(\"Process file: \", fname_in)\n",
- " dones.append(mapped_files[qm].empty())\n",
- " else:\n",
- " continue\n",
- " inp.append((fname_in, i, gg))\n",
- "\n",
- "with multiprocessing.Pool() as pool:\n",
- " results = pool.starmap(characterize_module, inp)\n",
- "\n",
- "for offset, noise, gains, gains_std, gg, bp, thiscell, thisacq in results:\n",
+ "for (_, module_index, gain_index), (offset, noise, gains, gains_std, bp,\n",
+ " thiscell, thisacq) in zip(inp, results):\n",
" all_cells.append(thiscell)\n",
" all_acq_rate.append(thisacq)\n",
- " for i in modules:\n",
- " qm = module_index_to_qm(i)\n",
- " if qm not in offset_g:\n",
- " offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
- " noise_g[qm] = np.zeros_like(offset_g[qm])\n",
- " badpix_g[qm] = np.zeros_like(offset_g[qm], np.uint32)\n",
- " if not fixed_gain_mode:\n",
- " gain_g[qm] = np.zeros_like(offset_g[qm])\n",
- " gainstd_g[qm] = np.zeros_like(offset_g[qm])\n",
- "\n",
- " offset_g[qm][...,gg] = offset\n",
- " noise_g[qm][...,gg] = noise\n",
- " badpix_g[qm][...,gg] = bp\n",
+ " qm = module_index_to_qm(module_index)\n",
+ " if qm not in offset_g:\n",
+ " offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
+ " noise_g[qm] = np.zeros_like(offset_g[qm])\n",
+ " badpix_g[qm] = np.zeros_like(offset_g[qm], np.uint32)\n",
" if not fixed_gain_mode:\n",
- " gain_g[qm][...,gg] = gains\n",
- " gainstd_g[qm][..., gg] = gains_std\n",
+ " gain_g[qm] = np.zeros_like(offset_g[qm])\n",
+ " gainstd_g[qm] = np.zeros_like(offset_g[qm])\n",
"\n",
+ " offset_g[qm][..., gain_index] = offset\n",
+ " noise_g[qm][..., gain_index] = noise\n",
+ " badpix_g[qm][..., gain_index] = bp\n",
+ " if not fixed_gain_mode:\n",
+ " gain_g[qm][..., gain_index] = gains\n",
+ " gainstd_g[qm][..., gain_index] = gains_std\n",
"\n",
- "duration = (datetime.now() - start).total_seconds()\n",
"\n",
"max_cells = np.max(all_cells)\n",
"print(f\"Using {max_cells} memory cells\")\n",
@@ -490,13 +505,16 @@
"metadata": {},
"outputs": [],
"source": [
- "# Add a badpixel due to bad gain separation\n",
+ "# Add bad pixels due to bad gain separation\n",
"if not fixed_gain_mode:\n",
- " for g in range(2):\n",
- " # Bad pixels during bad gain separation.\n",
- " # Fraction of pixels in the module with separation lower than \"thresholds_gain_sigma\".\n",
- " bad_sep = (gain_g[qm][..., g+1] - gain_g[qm][..., g]) / np.sqrt(gainstd_g[qm][..., g+1]**2 + gainstd_g[qm][..., g]**2)\n",
- " badpix_g[qm][...,g+1][(bad_sep)<thresholds_gain_sigma]|= BadPixels.GAIN_THRESHOLDING_ERROR.value"
+ " for qm in gain_g.keys():\n",
+ " for g in range(2):\n",
+ " # Bad pixels during bad gain separation.\n",
+ " # Fraction of pixels in the module with separation lower than \"thresholds_gain_sigma\".\n",
+ " bad_sep = (gain_g[qm][..., g+1] - gain_g[qm][..., g]) / \\\n",
+ " np.sqrt(gainstd_g[qm][..., g+1]**2 + gainstd_g[qm][..., g]**2)\n",
+ " badpix_g[qm][...,g+1][bad_sep<thresholds_gain_sigma] |= \\\n",
+ " BadPixels.GAIN_THRESHOLDING_ERROR"
]
},
{
@@ -553,23 +571,6 @@
"report = get_report(out_folder)"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# TODO: add db_module when received from myMDC\n",
- "# Create the modules dict of karabo_das and PDUs\n",
- "qm_dict = OrderedDict()\n",
- "for i, k_da in zip(modules, karabo_da):\n",
- " qm = module_index_to_qm(i)\n",
- " qm_dict[qm] = {\n",
- " \"karabo_da\": k_da,\n",
- " \"db_module\": \"\"\n",
- " }"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -578,11 +579,13 @@
"source": [
"# set the operating condition\n",
"# note: iCalibrationDB only adds gain_mode if it is truthy, so we don't need to handle None\n",
- "condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
- " bias_voltage=bias_voltage,\n",
- " acquisition_rate=acq_rate,\n",
- " gain_setting=gain_setting,\n",
- " gain_mode=fixed_gain_mode)"
+ "condition = iCalibrationDB.Conditions.Dark.AGIPD(\n",
+ " memory_cells=max_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " acquisition_rate=acq_rate,\n",
+ " gain_setting=gain_setting,\n",
+ " gain_mode=fixed_gain_mode\n",
+ ")"
]
},
{
@@ -591,45 +594,26 @@
"metadata": {},
"outputs": [],
"source": [
- "# Retrieve existing constants for comparison\n",
- "old_const = {}\n",
- "old_mdata = {}\n",
- "\n",
- "print('Retrieve pre-existing constants for comparison.')\n",
- "for qm in res:\n",
- " qm_db = qm_dict[qm]\n",
- " karabo_da = qm_db[\"karabo_da\"]\n",
- " for const in res[qm]:\n",
- " dconst = getattr(Constants.AGIPD, const)()\n",
- "\n",
- " # This should be used in case of running notebook\n",
- " # by a different method other than myMDC which already\n",
- " # sends CalCat info.\n",
- " # TODO: Set db_module to \"\" by default in the first cell\n",
- " if not qm_db[\"db_module\"]:\n",
- " qm_db[\"db_module\"] = get_pdu_from_db(karabo_id, karabo_da, dconst,\n",
- " condition, cal_db_interface,\n",
- " snapshot_at=creation_time)[0]\n",
- "\n",
- " data, mdata = get_from_db(karabo_id, karabo_da,\n",
- " dconst, condition,\n",
- " None, cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " verbosity=2, timeout=cal_db_timeout)\n",
- "\n",
- " old_const[const] = data\n",
- " if mdata is not None and data is not None:\n",
- " time = mdata.calibration_constant_version.begin_at\n",
- " old_mdata[const] = time.isoformat()\n",
- " os.makedirs('{}/old/'.format(out_folder), exist_ok=True)\n",
- " save_const_to_h5(qm_db[\"db_module\"], karabo_id,\n",
- " dconst, condition, data,\n",
- " file_loc, report, creation_time,\n",
- " f'{out_folder}/old/')\n",
- " else:\n",
- " old_mdata[const] = \"Not found\"\n",
- " with open(f\"{out_folder}/module_mapping_{qm}.yml\",\"w\") as fd:\n",
- " yaml.safe_dump({\"module_mapping\": {qm: qm_db[\"db_module\"]}}, fd)"
+ "# Create mapping from module(s) (qm) to karabo_da(s) and PDU(s)\n",
+ "qm_dict = OrderedDict()\n",
+ "all_pdus = get_pdu_from_db(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " constant=iCalibrationDB.CalibrationConstant(),\n",
+ " condition=condition,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " snapshot_at=creation_time.isoformat(),\n",
+ " timeout=cal_db_timeout\n",
+ ")\n",
+ "for module_index, module_da, module_pdu in zip(modules, karabo_da, all_pdus):\n",
+ " qm = module_index_to_qm(module_index)\n",
+ " qm_dict[qm] = {\n",
+ " \"karabo_da\": module_da,\n",
+ " \"db_module\": module_pdu\n",
+ " }\n",
+ " # saving mapping information for summary notebook\n",
+ " with open(f\"{out_folder}/module_mapping_{qm}.yml\", \"w\") as fd:\n",
+ " yaml.safe_dump({\"module_mapping\": {qm: module_pdu}}, fd)"
]
},
{
@@ -641,10 +625,9 @@
"md = None\n",
"\n",
"for qm in res:\n",
- " karabo_da = qm_dict[qm][\"karabo_da\"]\n",
" db_module = qm_dict[qm][\"db_module\"]\n",
" for const in res[qm]:\n",
- " dconst = getattr(Constants.AGIPD, const)()\n",
+ " dconst = getattr(iCalibrationDB.Constants.AGIPD, const)()\n",
" dconst.data = res[qm][const]\n",
"\n",
" if db_output:\n",
@@ -655,7 +638,7 @@
" if local_output:\n",
" md = save_const_to_h5(db_module, karabo_id, dconst, condition, dconst.data,\n",
" file_loc, report, creation_time, out_folder)\n",
- " print(f\"Calibration constant {const} is stored locally.\\n\")\n",
+ " print(f\"Calibration constant {const} for {qm} is stored locally in {file_loc}.\\n\")\n",
"\n",
" print(\"Constants parameter conditions are:\\n\")\n",
" print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
@@ -664,6 +647,50 @@
" f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Start retrieving existing constants for comparison\n",
+ "qm_x_const = [(qm, const) for const in res[qm] for qm in res]\n",
+ "\n",
+ "\n",
+ "def retrieve_old_constant(qm, const):\n",
+ " dconst = getattr(iCalibrationDB.Constants.AGIPD, const)()\n",
+ "\n",
+ " # This should be used in case of running notebook\n",
+ " # by a different method other than myMDC which already\n",
+ " # sends CalCat info.\n",
+ " # TODO: Set db_module to \"\" by default in the first cell\n",
+ "\n",
+ " data, mdata = get_from_db(\n",
+ " karabo_id,\n",
+ " qm_dict[qm][\"karabo_da\"],\n",
+ " constant=dconst,\n",
+ " condition=condition,\n",
+ " empty_constant=None,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " verbosity=2,\n",
+ " timeout=cal_db_timeout\n",
+ " )\n",
+ "\n",
+ " if mdata is None or data is None:\n",
+ " timestamp = \"Not found\"\n",
+ " else:\n",
+ " timestamp = mdata.calibration_constant_version.begin_at.isoformat()\n",
+ " \n",
+ " return data, timestamp\n",
+ "\n",
+ "old_retrieval_pool = multiprocessing.Pool()\n",
+ "old_retrieval_res = old_retrieval_pool.starmap_async(\n",
+ " retrieve_old_constant, qm_x_const\n",
+ ")\n",
+ "old_retrieval_pool.close()"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -674,7 +701,7 @@
"for i in modules:\n",
" qm = module_index_to_qm(i)\n",
" mnames.append(qm)\n",
- " display(Markdown(f'## Position of the module {qm} and its ASICs##'))\n",
+ " display(Markdown(f'## Position of the module {qm} and its ASICs'))\n",
"show_processed_modules(dinstance, constants=None, mnames=mnames, mode=\"position\")"
]
},
@@ -747,15 +774,17 @@
"metadata": {},
"outputs": [],
"source": [
- "cols = {BadPixels.NOISE_OUT_OF_THRESHOLD.value: (BadPixels.NOISE_OUT_OF_THRESHOLD.name, '#FF000080'),\n",
- " BadPixels.OFFSET_NOISE_EVAL_ERROR.value: (BadPixels.OFFSET_NOISE_EVAL_ERROR.name, '#0000FF80'),\n",
- " BadPixels.OFFSET_OUT_OF_THRESHOLD.value: (BadPixels.OFFSET_OUT_OF_THRESHOLD.name, '#00FF0080'),\n",
- " BadPixels.GAIN_THRESHOLDING_ERROR.value: (BadPixels.GAIN_THRESHOLDING_ERROR.name, '#FF40FF40'),\n",
- " BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value: ('OFFSET_OUT_OF_THRESHOLD + NOISE_OUT_OF_THRESHOLD', '#DD00DD80'),\n",
- " BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value |\n",
- " BadPixels.GAIN_THRESHOLDING_ERROR.value: ('MIXED', '#BFDF009F')}\n",
- "\n",
"if high_res_badpix_3d:\n",
+ " cols = {\n",
+ " BadPixels.NOISE_OUT_OF_THRESHOLD: (BadPixels.NOISE_OUT_OF_THRESHOLD.name, '#FF000080'),\n",
+ " BadPixels.OFFSET_NOISE_EVAL_ERROR: (BadPixels.OFFSET_NOISE_EVAL_ERROR.name, '#0000FF80'),\n",
+ " BadPixels.OFFSET_OUT_OF_THRESHOLD: (BadPixels.OFFSET_OUT_OF_THRESHOLD.name, '#00FF0080'),\n",
+ " BadPixels.GAIN_THRESHOLDING_ERROR: (BadPixels.GAIN_THRESHOLDING_ERROR.name, '#FF40FF40'),\n",
+ " BadPixels.OFFSET_OUT_OF_THRESHOLD | BadPixels.NOISE_OUT_OF_THRESHOLD: ('OFFSET_OUT_OF_THRESHOLD + NOISE_OUT_OF_THRESHOLD', '#DD00DD80'),\n",
+ " BadPixels.OFFSET_OUT_OF_THRESHOLD | BadPixels.NOISE_OUT_OF_THRESHOLD |\n",
+ " BadPixels.GAIN_THRESHOLDING_ERROR: ('MIXED', '#BFDF009F')\n",
+ " }\n",
+ "\n",
" display(Markdown(\"\"\"\n",
"\n",
" ## Global Bad Pixel Behaviour ##\n",
@@ -819,7 +848,6 @@
" bp_thresh[mod][...,:2] = con[...,:2]\n",
" bp_thresh[mod][...,2:] = con\n",
"\n",
- "\n",
" create_constant_overview(thresholds_g, \"Threshold (ADU)\", max_cells, 4000, 10000, 5,\n",
" badpixels=[bp_thresh, np.nan],\n",
" gmap=['HG-MG Threshold', 'MG-LG Threshold', 'High gain', 'Medium gain', 'low gain'],\n",
@@ -854,9 +882,27 @@
"metadata": {},
"outputs": [],
"source": [
- "display(Markdown('The following pre-existing constants are used for comparison: \\n'))\n",
- "for key in old_mdata:\n",
- " display(Markdown('**{}** at {}'.format(key, old_mdata[key])))"
+ "# now we need the old constants\n",
+ "old_const = {}\n",
+ "old_mdata = {}\n",
+ "old_retrieval_res.wait()\n",
+ "\n",
+ "for (qm, const), (data, timestamp) in zip(qm_x_const, old_retrieval_res.get()):\n",
+ " old_const.setdefault(qm, {})[const] = data\n",
+ " old_mdata.setdefault(qm, {})[const] = timestamp"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown(\"The following pre-existing constants are used for comparison:\"))\n",
+ "for qm, consts in old_mdata.items():\n",
+ " display(Markdown(f\"- {qm}\"))\n",
+ " for const in consts:\n",
+ " display(Markdown(f\" - {const} at {consts[const]}\"))"
]
},
{
@@ -870,7 +916,6 @@
"bits = [BadPixels.NOISE_OUT_OF_THRESHOLD, BadPixels.OFFSET_OUT_OF_THRESHOLD, BadPixels.OFFSET_NOISE_EVAL_ERROR, BadPixels.GAIN_THRESHOLDING_ERROR]\n",
"for qm in badpix_g.keys():\n",
" for gain in range(3):\n",
- "\n",
" l_data = []\n",
" l_data_old = []\n",
"\n",
@@ -878,14 +923,14 @@
" datau32 = data.astype(np.uint32)\n",
" l_data.append(len(datau32[datau32>0].flatten()))\n",
" for bit in bits:\n",
- " l_data.append(np.count_nonzero(badpix_g[qm][:,:,:,gain] & bit.value))\n",
+ " l_data.append(np.count_nonzero(badpix_g[qm][:,:,:,gain] & bit))\n",
"\n",
- " if old_const['BadPixelsDark'] is not None:\n",
- " dataold = np.copy(old_const['BadPixelsDark'][:, :, :, gain])\n",
+ " if old_const[qm]['BadPixelsDark'] is not None:\n",
+ " dataold = np.copy(old_const[qm]['BadPixelsDark'][:, :, :, gain])\n",
" datau32old = dataold.astype(np.uint32)\n",
" l_data_old.append(len(datau32old[datau32old>0].flatten()))\n",
" for bit in bits:\n",
- " l_data_old.append(np.count_nonzero(old_const['BadPixelsDark'][:, :, :, gain] & bit.value))\n",
+ " l_data_old.append(np.count_nonzero(old_const[qm]['BadPixelsDark'][:, :, :, gain] & bit))\n",
"\n",
" l_data_name = ['All bad pixels', 'NOISE_OUT_OF_THRESHOLD',\n",
" 'OFFSET_OUT_OF_THRESHOLD', 'OFFSET_NOISE_EVAL_ERROR', 'GAIN_THRESHOLDING_ERROR']\n",
@@ -897,7 +942,7 @@
" for i in range(len(l_data)):\n",
" line = [f'{l_data_name[i]}, {gain_names[gain]} gain', l_threshold[i], l_data[i]]\n",
"\n",
- " if old_const['BadPixelsDark'] is not None:\n",
+ " if old_const[qm]['BadPixelsDark'] is not None:\n",
" line += [l_data_old[i]]\n",
" else:\n",
" line += ['-']\n",
@@ -906,8 +951,7 @@
" table.append(['', '', '', ''])\n",
"\n",
"display(Markdown('''\n",
- "\n",
- "### Number of bad pixels ###\n",
+ "### Number of bad pixels\n",
"\n",
"One pixel can be bad for different reasons, therefore, the sum of all types of bad pixels can be more than the number of all bad pixels.\n",
"\n",
@@ -935,49 +979,67 @@
"else:\n",
" constants = ['Offset', 'Noise', 'ThresholdsDark']\n",
"\n",
- "for const in constants:\n",
+ "constants_x_qms = list(itertools.product(constants, res.keys()))\n",
+ "\n",
"\n",
+ "def compute_table(const, qm):\n",
" if const == 'ThresholdsDark':\n",
" table = [['','HG-MG threshold', 'HG-MG threshold', 'MG-LG threshold', 'MG-LG threshold']]\n",
" else:\n",
" table = [['','High gain', 'High gain', 'Medium gain', 'Medium gain', 'Low gain', 'Low gain']]\n",
"\n",
- " for qm in res.keys():\n",
- " data = np.copy(res[qm][const])\n",
+ " compare_with_old_constant = old_const[qm][const] is not None and \\\n",
+ " old_const[qm]['BadPixelsDark'] is not None\n",
+ "\n",
+ " data = np.copy(res[qm][const])\n",
+ "\n",
+ " if const == 'ThresholdsDark':\n",
+ " data[...,0][res[qm]['BadPixelsDark'][...,0]>0] = np.nan\n",
+ " data[...,1][res[qm]['BadPixelsDark'][...,1]>0] = np.nan\n",
+ " else:\n",
+ " data[res[qm]['BadPixelsDark']>0] = np.nan\n",
"\n",
+ " if compare_with_old_constant:\n",
+ " data_old = np.copy(old_const[qm][const])\n",
" if const == 'ThresholdsDark':\n",
- " data[...,0][res[qm]['BadPixelsDark'][...,0]>0] = np.nan\n",
- " data[...,1][res[qm]['BadPixelsDark'][...,1]>0] = np.nan\n",
+ " data_old[...,0][old_const[qm]['BadPixelsDark'][...,0]>0] = np.nan\n",
+ " data_old[...,1][old_const[qm]['BadPixelsDark'][...,1]>0] = np.nan\n",
" else:\n",
- " data[res[qm]['BadPixelsDark']>0] = np.nan\n",
- "\n",
- " if old_const[const] is not None and old_const['BadPixelsDark'] is not None:\n",
- " dataold = np.copy(old_const[const])\n",
- " if const == 'ThresholdsDark':\n",
- " dataold[...,0][old_const['BadPixelsDark'][...,0]>0] = np.nan\n",
- " dataold[...,1][old_const['BadPixelsDark'][...,1]>0] = np.nan\n",
+ " data_old[old_const[qm]['BadPixelsDark']>0] = np.nan\n",
+ "\n",
+ " f_list = [np.nanmedian, np.nanmean, np.nanstd, np.nanmin, np.nanmax]\n",
+ " n_list = ['Median', 'Mean', 'Std', 'Min', 'Max']\n",
+ "\n",
+ " def compute_row(i):\n",
+ " line = [n_list[i]]\n",
+ " for gain in range(3):\n",
+ " # Compare only 3 threshold gain-maps\n",
+ " if gain == 2 and const == 'ThresholdsDark':\n",
+ " continue\n",
+ " stat_measure = f_list[i](data[...,gain])\n",
+ " line.append(f\"{stat_measure:6.1f}\")\n",
+ " if compare_with_old_constant:\n",
+ " old_stat_measure = f_list[i](data_old[...,gain])\n",
+ " line.append(f\"{old_stat_measure:6.1f}\")\n",
" else:\n",
- " dataold[old_const['BadPixelsDark']>0] = np.nan\n",
- "\n",
- " f_list = [np.nanmedian, np.nanmean, np.nanstd, np.nanmin, np.nanmax]\n",
- " n_list = ['Median', 'Mean', 'Std', 'Min', 'Max']\n",
- "\n",
- " for i, f in enumerate(f_list):\n",
- " line = [n_list[i]]\n",
- " for gain in range(3):\n",
- " # Compare only 3 threshold gain-maps\n",
- " if gain == 2 and const == 'ThresholdsDark':\n",
- " continue\n",
- " line.append('{:6.1f}'.format(f(data[...,gain])))\n",
- " if old_const[const] is not None and old_const['BadPixelsDark'] is not None:\n",
- " line.append('{:6.1f}'.format(f(dataold[...,gain])))\n",
- " else:\n",
- " line.append('-')\n",
+ " line.append(\"-\")\n",
+ " return line\n",
+ " \n",
"\n",
- " table.append(line)\n",
+ " with multiprocessing.pool.ThreadPool(processes=multiprocessing.cpu_count() // len(constants_x_qms)) as pool:\n",
+ " rows = pool.map(compute_row, range(len(f_list)))\n",
+ "\n",
+ " table.extend(rows)\n",
+ "\n",
+ " return table\n",
+ "\n",
+ "\n",
+ "with multiprocessing.Pool(processes=len(constants_x_qms)) as pool:\n",
+ " tables = pool.starmap(compute_table, constants_x_qms)\n",
"\n",
- " display(Markdown('### {} [ADU], good pixels only ###'.format(const)))\n",
- " md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header)))"
+ "for (const, qm), table in zip(constants_x_qms, tables):\n",
+ " display(Markdown(f\"### {qm}: {const} [ADU], good pixels only\"))\n",
+ " display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header)))"
]
}
],
diff --git a/setup.py b/setup.py
index c5eb537959cf1afbcc6613bb88c4fc75c0c0d07d..3ad381ac0a111e758849cbc05e2ad3a9819e4b8a 100644
--- a/setup.py
+++ b/setup.py
@@ -75,6 +75,7 @@ setup(
install_requires=[
"iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.0.5", # noqa
"XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.5.6-2.10.0#subdirectory=lib", # noqa
+ "pasha @ git+https://github.com/European-XFEL/pasha.git@0.1.0",
"Cython==0.29.21",
"Jinja2==2.11.2",
"astcheck==0.2.5",