diff --git a/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
index 1a3a829012d659b239715c1727a933e7c03600f7..c80308496724f7d00fd6a0c627fffcf5ae7f5f72 100644
--- a/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
@@ -163,12 +163,12 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 23,
"id": "ac9c5dc3-bc66-4e7e-b6a1-360259be535c",
"metadata": {},
"outputs": [],
"source": [
- "def specifiy_trains_to_process(\n",
+ "def specify_trains_to_process(\n",
" img_key_data: \"extra_data.KeyData\", # noqa\n",
" max_trains: int = 0,\n",
" min_trains: int = 0,\n",
@@ -214,63 +214,191 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 32,
"id": "4e8ffeae",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "DA01 has 1042 trains with empty frames out of 2041 trains\n",
+ "Processing 999 trains.\n",
+ "preparing raw data: 0.0 s\n",
+ "convert to 10bit: 4.3 s\n",
+ "Processing darks: 12.1 s\n"
+ ]
+ }
+ ],
"source": [
"# Calculate noise and offset per pixel and global average, std and median\n",
- "\n",
"noise_map = dict()\n",
"offset_map = dict()\n",
- "bad_pixels_map = dict()\n",
+ "badpixels_map = dict()\n",
"\n",
"context = psh.context.ProcessContext(num_workers=multiprocessing.cpu_count())\n",
"\n",
- "\n",
"for mod in karabo_da:\n",
" # Path to pixels ADC values\n",
" instr_mod_src = instrument_src.format(int(mod[-2:]))\n",
- " pixels_src = (instr_mod_src, \"data.adc\")\n",
" data_path = \"INSTRUMENT/\"+instr_mod_src+\"/data\"\n",
"\n",
" # TODO: Validate the final shape to store constants.\n",
" cshape = (3, 2, 1280)\n",
" offset_map[mod] = np.zeros(cshape, dtype=np.float32)\n",
" noise_map[mod] = np.zeros_like(offset_map[mod])\n",
- " bad_pixels_map[mod] = np.zeros_like(offset_map[mod], dtype=np.uint32)\n",
+ " badpixels_map[mod] = np.zeros_like(offset_map[mod], dtype=np.uint32)\n",
"\n",
" for run_num, [gain, run_dc] in run_dcs_dict.items():\n",
" step_timer.start()\n",
- " n_trains = specifiy_trains_to_process(run_dc[pixels_src])\n",
+ " n_trains = specify_trains_to_process(run_dc[instr_mod_src, \"data.adc\"])\n",
"\n",
" # Select requested number of trains to process.\n",
- " dc = run_dc.select(*pixels_src, require_all=True).select_trains(np.s_[:n_trains]) # noqa\n",
- " dshape = dc[instr_mod_src, \"data.adc\"].shape\n",
+ " dc = run_dc.select(\n",
+ " instr_mod_src, require_all=True).select_trains(np.s_[:n_trains]) # noqa\n",
+ "\n",
" step_timer.done_step(\"preparing raw data\")\n",
"\n",
" step_timer.start()\n",
" # Convert 12bit data to 10bit\n",
- " data_10bit = context.alloc(shape=dshape, dtype=np.float32)\n",
+ " data_10bit = context.alloc(\n",
+ " shape=dc[instr_mod_src, \"data.adc\"].shape,\n",
+ " dtype=np.float32\n",
+ " )\n",
" context.map(convert_train, dc)\n",
" step_timer.done_step(\"convert to 10bit\")\n",
"\n",
" step_timer.start()\n",
+ "\n",
+ " # Split even and odd data to calculate the two storage cell constants.\n",
" even_data = data_10bit[:, ::2, :]\n",
" odd_data = data_10bit[:, 1::2, :]\n",
- " \n",
+ " data_gain = dc[instr_mod_src, \"data.gain\"].ndarray()\n",
+ " even_gain = data_gain[:, ::2, :]\n",
+ " odd_gain = data_gain[:, 1::2, :]\n",
+ "\n",
" def offset_noise_cell(wid, index, d):\n",
- " offset[index] = np.mean(d, axis=(0, 1))\n",
- " noise[index] = np.std(d, axis=(0, 1))\n",
+ " offset_map[mod][gain, index, ...] = np.mean(d, axis=(0, 1))\n",
+ " noise_map[mod][gain, index, ...] = np.std(d, axis=(0, 1))\n",
"\n",
- " offset = context.alloc(shape=cshape[-2:], dtype=np.float32)\n",
- " noise = context.alloc(like=offset)\n",
+ " offset_map[mod] = context.alloc(shape=cshape, dtype=np.float32)\n",
+ " noise_map[mod] = context.alloc(like=offset_map[mod])\n",
+ " badpixels_map[mod] = context.alloc(like=offset_map[mod], dtype=np.int32)\n",
" context.map(offset_noise_cell, (even_data, odd_data))\n",
- " offset_map[mod][gain, ...] = offset.copy()\n",
- " noise_map[mod][gain, ...] = noise.copy()\n",
"\n",
- " step_timer.done_step(\"Processing darks\")\n",
- " "
+ " raw_g = 3 if gain == 2 else gain\n",
+ "\n",
+ " def badpixels_cell(wid, index, g):\n",
+ " \"\"\"Check if there are wrong bad gain values.\n",
+ " Indicate pixels with wrong gain value across all trains for each cell.\"\"\"\n",
+ " badpixels_map[mod][gain, index,\n",
+ " np.mean(g, axis=(0, 1)) != raw_g] |= BadPixels.WRONG_GAIN_VALUE.value\n",
+ " context.map(badpixels_cell, (even_gain, odd_gain))\n",
+ "\n",
+ " step_timer.done_step(\"Processing darks\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 42,
+ "id": "4b81a3e7",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(3, 2, 1280)"
+ ]
+ },
+ "execution_count": 42,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "badpixels_map[mod].shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "id": "7ec0e569",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([], dtype=uint8)"
+ ]
+ },
+ "execution_count": 43,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.where(badpixels_map[mod] != 0])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3fc17e05-17ab-4ac4-9e79-c95399278bb9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def print_bp_entry(bp):\n",
+ " print(\"{:<30s} {:032b} -> {}\".format(bp.name, bp.value, int(bp.value)))\n",
+ "\n",
+ "print_bp_entry(BadPixels.OFFSET_OUT_OF_THRESHOLD)\n",
+ "print_bp_entry(BadPixels.NOISE_OUT_OF_THRESHOLD)\n",
+ "print_bp_entry(BadPixels.OFFSET_NOISE_EVAL_ERROR)\n",
+ "print_bp_entry(BadPixels.WRONG_GAIN_VALUE)\n",
+ "\n",
+ "def eval_bpidx(d):\n",
+ "\n",
+ " mdn = np.nanmedian(d, axis=(0, 1))[None, None, :, :]\n",
+ " std = np.nanstd(d, axis=(0, 1))[None, None, :, :] \n",
+ " idx = (d > badpixel_threshold_sigma*std+mdn) | (d < (-badpixel_threshold_sigma)*std+mdn)\n",
+ "\n",
+ " return idx"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "40c34cc5-fe93-4b83-bf39-f465f37c40b4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "\n",
+ "for mod in karabo_da:\n",
+ " display(Markdown(f\"### Badpixels for module {mod}:\"))\n",
+ " offset_abs_threshold = np.array(offset_abs_threshold)\n",
+ "\n",
+ " bad_pixels_map[mod][eval_bpidx(offset_map[mod])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
+ "\n",
+ " bad_pixels_map[mod][~np.isfinite(offset_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
+ "\n",
+ " bad_pixels_map[mod][eval_bpidx(noise_map[mod])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
+ "\n",
+ " bad_pixels_map[mod][~np.isfinite(noise_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
+ "\n",
+ " bad_pixels_map[mod][(offset_map[mod] < offset_abs_threshold[0][None, None, None, :]) | (offset_map[mod] > offset_abs_threshold[1][None, None, None, :])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value # noqa\n",
+ "\n",
+ " for g_idx in gains:\n",
+ " for cell in range(memory_cells):\n",
+ " bad_pixels = bad_pixels_map[mod][:, :, cell, g_idx]\n",
+ " fn_0 = heatmapPlot(\n",
+ " np.swapaxes(bad_pixels, 0, 1),\n",
+ " y_label=\"Row\",\n",
+ " x_label=\"Column\",\n",
+ " lut_label=f\"Badpixels {g_name[g_idx]} [ADCu]\",\n",
+ " aspect=1.,\n",
+ " vmin=0, vmax=5,\n",
+ " title=f'G{g_idx} Bad pixel map - Cell {cell:02d} - Module {mod}')\n",
+ "step_timer.done_step(f'Creating bad pixels constant and plotting it.')"
]
},
{