diff --git a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
index 67b052ab358ec5f3fbb678fa062982c932ed09ba..0b5f1c966a23d46ac014612c967240be4e314c05 100644
--- a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
+++ b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
@@ -10,9 +10,9 @@
"\n",
"# Summary for processed of dark calibration constants and a comparison with previous injected constants.\n",
"\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/fixed_gain/SPB_summary_fix2\" # path to output to, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/kluyvert/lpd-dark-p900320-r26_27_28\" # path to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
- "karabo_id = \"SPB_DET_AGIPD1M-1\" # detector instance\n",
+ "karabo_id = \"FXE_DET_LPD1M-1\" # detector instance\n",
"gain_names = ['High gain', 'Medium gain', 'Low gain'] # a list of gain names to be used in plotting\n",
"threshold_names = ['HG-MG threshold', 'MG_LG threshold'] # a list of gain names to be used in plotting\n",
"local_output = True # Boolean indicating that local constants were stored in the out_folder\n",
@@ -43,6 +43,7 @@
"import h5py\n",
"import matplotlib\n",
"import numpy as np\n",
+ "import pasha as psh\n",
"import yaml\n",
"from IPython.display import Latex, Markdown, display\n",
"\n",
@@ -175,7 +176,7 @@
"elif \"DSSC\" in karabo_id:\n",
" dinstance = \"DSSC1M1\"\n",
" nmods = 16\n",
- " expected_constants = ['Offset', 'Noise', 'BadPixelsDark']\n",
+ " expected_constants = ['Offset', 'Noise']\n",
" display(Markdown(\"\"\"\n",
" \n",
"# Summary of DSSC dark characterization #\n",
@@ -199,11 +200,34 @@
" module = fdict[\"module\"]\n",
" mod_mapping[module] = fdict[\"pdu\"]\n",
" old_constant_metadata[module] = fdict[\"old-constants\"]\n",
- " fn.unlink()\n",
"\n",
"metadata.save()"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# In AGIPD fixed gain mode, ThresholdsDark is not expected\n",
+ "if 'AGIPD' in karabo_id:\n",
+ " for i in range(nmods):\n",
+ " qm = module_index_to_qm(i)\n",
+ " if not mod_mapping.get(qm):\n",
+ " continue\n",
+ " mod_pdu = mod_mapping[qm]\n",
+ " fpath = out_folder / f\"const_Offset_{mod_pdu}.h5\"\n",
+ " if not fpath.exists():\n",
+ " continue\n",
+ "\n",
+ " with h5py.File(fpath, 'r') as f:\n",
+ " if 'Gain mode' in f['condition']:\n",
+ " if f[\"condition\"][\"Gain mode\"][\"value\"][()]:\n",
+ " expected_constants.remove(\"ThresholdsDark\")\n",
+ " break"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -217,57 +241,54 @@
"metadata": {},
"outputs": [],
"source": [
- "# TODO: After changes in the Cal DB interface read files from cal repository\n",
- "# Load constants from local files\n",
- "data = OrderedDict()\n",
- "old_cons = OrderedDict()\n",
- "mod_names = []\n",
- "\n",
- "gain_mode = None if \"AGIPD\" in karabo_id else False\n",
- "\n",
- "# Loop over modules\n",
- "for i in range(nmods):\n",
- " qm = module_index_to_qm(i)\n",
- " if not mod_mapping.get(qm):\n",
- " continue\n",
- " mod_pdu = mod_mapping[qm]\n",
- " # Loop over expected dark constants in out_folder.\n",
- " # Some constants can be missing in out_folder.\n",
- " # e.g. ThresholdsDark for fixed gain AGIPD, DSSC and LPD.\n",
- " for const in expected_constants:\n",
- " # first load new constant\n",
- " fpath = out_folder / f\"const_{const}_{mod_pdu}.h5\"\n",
- " # No ThresholdsDark expected for AGIPD in fixed gain mode.\n",
- " if const == \"ThresholdsDark\" and gain_mode:\n",
+ "# Get shape, dtype, and number of files for each constant.\n",
+ "# Also build lists of the files involved, to be loaded in parallel in a later cell.\n",
+ "const_shape_and_dtype = {}\n",
+ "found_module_nums = set()\n",
+ "pieces_to_load = []\n",
+ "pieces_to_load_prev = []\n",
+ "\n",
+ "for cname in expected_constants:\n",
+ " for i in range(nmods):\n",
+ " qm = module_index_to_qm(i)\n",
+ " if not mod_mapping.get(qm):\n",
" continue\n",
+ " mod_pdu = mod_mapping[qm]\n",
+ " fpath = out_folder / f\"const_{cname}_{mod_pdu}.h5\"\n",
" if not fpath.exists():\n",
- " print(f\"No local output file {fpath} found\")\n",
" continue\n",
- "\n",
- " with h5py.File(fpath, 'r') as f:\n",
- " if qm not in data:\n",
- " mod_names.append(qm)\n",
- " data.setdefault(qm, OrderedDict())[const] = f[\"data\"][()]\n",
- "\n",
- " if gain_mode is None:\n",
- " if 'Gain Mode' in f['condition'].keys():\n",
- " gain_mode = bool(f[\"condition\"][\"Gain Mode\"][\"value\"][()])\n",
- " else:\n",
- " gain_mode = False\n",
+ " \n",
+ " pieces_to_load.append((cname, i, fpath))\n",
+ " found_module_nums.add(i)\n",
+ " \n",
" # try finding old constants using paths from CalCat store\n",
+ " if qm not in old_constant_metadata:\n",
+ " continue\n",
" qm_mdata = old_constant_metadata[qm]\n",
"\n",
- " if const not in qm_mdata:\n",
+ " if cname not in qm_mdata:\n",
" continue\n",
"\n",
- " filepath = qm_mdata[const][\"filepath\"]\n",
- " h5path = qm_mdata[const][\"h5path\"]\n",
+ " fpath_prev = qm_mdata[cname][\"filepath\"]\n",
+ " h5path_prev = qm_mdata[cname][\"h5path\"]\n",
"\n",
- " if not filepath or not h5path:\n",
- " continue\n",
- "\n",
- " with h5py.File(filepath, \"r\") as fd:\n",
- " old_cons.setdefault(qm, OrderedDict())[const] = fd[f\"{h5path}/data\"][:]"
+ " if fpath_prev and h5path_prev:\n",
+ " pieces_to_load_prev.append((cname, i, fpath_prev, h5path_prev))\n",
+ " \n",
+ " # Get the constant shape from one of the module files\n",
+ " with h5py.File(fpath, 'r') as f:\n",
+ " const_shape_and_dtype[cname] = (f['data'].shape, f['data'].dtype)\n",
+ " \n",
+ "# Allocate arrays for these constants (without space for missing modules)\n",
+ "nmods_found = len(found_module_nums)\n",
+ "constants = {\n",
+ " cname: psh.alloc((nmods_found,) + module_const_shape, dtype=dt, fill=0)\n",
+ " for cname, (module_const_shape, dt) in const_shape_and_dtype.items()\n",
+ "}\n",
+ "prev_const = {\n",
+ " cname: psh.alloc((nmods_found,) + module_const_shape, dtype=dt, fill=0)\n",
+ " for cname, (module_const_shape, dt) in const_shape_and_dtype.items()\n",
+ "}"
]
},
{
@@ -276,28 +297,29 @@
"metadata": {},
"outputs": [],
"source": [
- "cons_shape = {}\n",
- "# extracting constant shape.\n",
- "for qm, constant in data.items():\n",
- " for cname, cons in constant.items():\n",
- " shape = data[qm][cname].shape\n",
- " if cname not in cons_shape:\n",
- " cons_shape[cname] = shape\n",
- " break\n",
- "\n",
- "constants = {}\n",
- "prev_const = {}\n",
- "\n",
- "for cname, sh in cons_shape.items():\n",
- " constants[cname]= np.zeros((len(mod_names),) + sh[:])\n",
- " prev_const[cname]= np.zeros((len(mod_names),) + sh[:])\n",
- "for i in range(len(mod_names)):\n",
- " for cname, cval in constants.items():\n",
- " cval[i] = data[mod_names[i]][cname]\n",
- " if mod_names[i] in old_cons.keys():\n",
- " prev_const[cname][i] = old_cons[mod_names[i]][cname]\n",
- " else:\n",
- " print(f\"No previous {cname} found for {mod_names[i]}\")"
+ "# Load the constant data in parallel\n",
+ "found_module_nums = sorted(found_module_nums)\n",
+ "mod_names = [module_index_to_qm(n) for n in found_module_nums]\n",
+ "\n",
+ "def load_piece(wid, ix, entry):\n",
+ " cname, mod_no, fpath = entry\n",
+ " mod_ix = found_module_nums.index(mod_no)\n",
+ " \n",
+ " with h5py.File(fpath, 'r') as f:\n",
+ " f['data'].read_direct(constants[cname][mod_ix])\n",
+ "\n",
+ "psh.map(load_piece, pieces_to_load)\n",
+ "print(f\"Loaded constant data from {len(pieces_to_load)} files\")\n",
+ "\n",
+ "def load_piece_prev(wid, ix, entry):\n",
+ " cname, mod_no, fpath, h5path = entry\n",
+ " mod_ix = found_module_nums.index(mod_no)\n",
+ " \n",
+ " with h5py.File(fpath, 'r') as f:\n",
+ " f[h5path]['data'].read_direct(prev_const[cname][mod_ix])\n",
+ "\n",
+ "psh.map(load_piece_prev, pieces_to_load_prev)\n",
+ "print(f\"Loaded previous constant data from {len(pieces_to_load_prev)} files\")"
]
},
{
@@ -330,32 +352,87 @@
" (-11.5, 8),\n",
" (254.5, -16),\n",
" (278.5, 275)])\n",
- " pixels_y = 256\n",
- " pixels_x = 256\n",
+ " module_shape = (256, 256)\n",
"\n",
"elif dinstance in ('AGIPD1M1', 'AGIPD1M2'):\n",
" geom = extra_geom.AGIPD_1MGeometry.from_quad_positions(quad_pos=[(-525, 625),\n",
" (-550, -10),\n",
" (520, -160),\n",
" (542.5, 475)])\n",
- " pixels_y = 128\n",
- " pixels_x = 512\n",
+ " module_shape = (512, 128)\n",
"\n",
"elif dinstance == \"AGIPD500K\":\n",
" geom = extra_geom.AGIPD_500K2GGeometry.from_origin()\n",
- " pixels_y = 128\n",
- " pixels_x = 512\n",
+ " module_shape = (512, 128)\n",
" \n",
"elif \"DSSC\" in dinstance:\n",
- " pixels_y = 512\n",
- " pixels_x = 128\n",
+ " module_shape = (128, 512)\n",
" quadpos = [(-130, 5), (-130, -125), (5, -125), (5, 5)]\n",
- "\n",
- " extrageom_pth = os.path.dirname(extra_geom.__file__)\n",
- " geom = extra_geom.DSSC_1MGeometry.from_h5_file_and_quad_positions(\n",
- " f\"{extrageom_pth}/tests/dssc_geo_june19.h5\", positions=quadpos)\n",
- "\n",
- "Mod_data=OrderedDict()\n",
+ " geom = extra_geom.DSSC_1MGeometry.from_quad_positions(quadpos)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def plot_const_and_delta(const, delta, const_name, gain_name):\n",
+ " gs = gridspec.GridSpec(2, 2)\n",
+ " fig = plt.figure(figsize=(24, 32))\n",
+ " \n",
+ " ax0 = fig.add_subplot(gs[0, :])\n",
+ " vmin, vmax = get_range(const, 2)\n",
+ " geom.plot_data_fast(\n",
+ " const, vmin=vmin, vmax=vmax, ax=ax0, colorbar={\n",
+ " 'shrink': 0.9, 'pad': 0.05, 'label': 'ADUs'\n",
+ " })\n",
+ " ax0.set_title(f\"{const_name} - {gain_name}\", fontsize=15)\n",
+ " \n",
+ " if np.count_nonzero(delta) == np.count_nonzero(np.isnan(delta)):\n",
+ " fig.text(0.5, 0.4, \"No difference from previous constant\",\n",
+ " ha='center', va='center', fontsize=15)\n",
+ " return\n",
+ " \n",
+ " # Plot delta from previous constant\n",
+ " ax1 = fig.add_subplot(gs[1, 0])\n",
+ " vmin, vmax = get_range(delta, 2)\n",
+ " vmax = max(vmax, abs(vmin)) # Center around zero\n",
+ " geom.plot_data_fast(\n",
+ " delta, vmin=-vmax, vmax=vmax, ax=ax1, cmap=\"RdBu\", colorbar={\n",
+ " 'shrink': 0.6, 'pad': 0.1, 'label': 'ADUs'\n",
+ " })\n",
+ " ax1.set_title(f\"Difference with previous {const_name} - {gain_name}\", fontsize=15)\n",
+ " \n",
+ " # Plot % delta from previous constant\n",
+ " delta_pct = delta / const * 100\n",
+ " ax2 = fig.add_subplot(gs[1, 1])\n",
+ " vmin, vmax = get_range(delta_pct, 2)\n",
+ " vmax = max(vmax, abs(vmin)) # Center around zero\n",
+ " geom.plot_data_fast(\n",
+ " delta_pct, vmin=-vmax, vmax=vmax, ax=ax2, cmap=\"RdBu\", colorbar={\n",
+ " 'shrink': 0.6, 'pad': 0.1, 'label': '%'\n",
+ " })\n",
+ " ax2.set_title(\"Percentage difference\", fontsize=15)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "psh_ctx = psh.ProcessContext(nmods)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
"gainstages = 1\n",
"\n",
"for const_name, const in constants.items():\n",
@@ -366,9 +443,6 @@
" gainstages = 3\n",
" else:\n",
" gainstages = 1\n",
- "\n",
- " for dname in ['{}', 'd-{}', 'dpct-{}']:\n",
- " Mod_data[dname.format(const_name)] = OrderedDict()\n",
" \n",
" display(Markdown(f'##### {const_name}'))\n",
" print_once = True\n",
@@ -376,72 +450,38 @@
" if const_name == 'ThresholdsDark':\n",
" if gain > 1:\n",
" continue\n",
- " glabel = threshold_names\n",
+ " glabel = threshold_names[gain]\n",
" else:\n",
- " glabel = gain_names\n",
- " for i in range(nmods):\n",
+ " glabel = gain_names[gain]\n",
+ "\n",
+ " stacked_const = psh_ctx.alloc((nmods,) + module_shape, dtype=np.float64, fill=0)\n",
+ " stacked_delta = psh_ctx.alloc((nmods,) + module_shape, dtype=np.float64, fill=0)\n",
+ " \n",
+ " def average_module(wid, i, _):\n",
" qm = module_index_to_qm(i)\n",
" if qm in mod_names:\n",
" m_idx = mod_names.index(qm)\n",
" # Check if constant shape of 5 indices e.g. AGIPD, LPD\n",
- " if len(const.shape) == 5:\n",
+ " if const.ndim == 5:\n",
" values = np.nanmean(const[m_idx, :, :, :, gain], axis=2)\n",
- " dval = np.nanmean(prev_const[const_name][m_idx, :, :, :, gain], axis=2)\n",
+ " prev_val = np.nanmean(prev_const[const_name][m_idx, :, :, :, gain], axis=2)\n",
" else:\n",
" values = np.nanmean(const[m_idx, :, :, :], axis=2)\n",
- " dval = np.nanmean(prev_const[const_name][m_idx, :, :, :], axis=2)\n",
+ " prev_val = np.nanmean(prev_const[const_name][m_idx, :, :, :], axis=2)\n",
" values[values == 0] = np.nan\n",
- " dval[dval == 0] = np.nan\n",
- " dval = values - dval\n",
- " dval_pct = dval/values * 100\n",
- "\n",
- " values = np.moveaxis(values, 0, -1).reshape(1, values.shape[1], values.shape[0])\n",
- " dval = np.moveaxis(dval, 0, -1).reshape(1, dval.shape[1], dval.shape[0])\n",
- " dval_pct = np.moveaxis(dval_pct, 0, -1).reshape(1, dval_pct.shape[1], dval_pct.shape[0])\n",
+ " prev_val[prev_val == 0] = np.nan\n",
+ " stacked_const[i] = np.moveaxis(values, 0, -1)\n",
+ " stacked_delta[i] = np.moveaxis(values - prev_val, 0, -1)\n",
" else:\n",
- " # if module not available fill arrays with nan\n",
- " values = np.zeros((1, pixels_x, pixels_y),dtype=np.float64)\n",
- " values[values == 0] = np.nan\n",
- " dval = values \n",
- " dval_pct = dval\n",
- " for k, v in {'{}': values, 'd-{}': dval , 'dpct-{}': dval_pct}.items():\n",
- " try:\n",
- " Mod_data[k.format(const_name)][gain_names[gain]] = \\\n",
- " np.concatenate((Mod_data[k.format(const_name)][gain_names[gain]],\n",
- " v), axis=0)\n",
- " except:\n",
- " Mod_data[k.format(const_name)][gain_names[gain]] = v\n",
- " if np.count_nonzero(dval) == 0 and print_once:\n",
- " display(Markdown(f'New and previous {const_name} are the same, hence there is no difference.'))\n",
- " print_once = False\n",
+ " # if module not available fill space with nan\n",
+ " stacked_const[i] = np.nan\n",
+ "\n",
+ " psh_ctx.map(average_module, range(nmods))\n",
+ " \n",
" # Plotting constant overall modules.\n",
- " display(Markdown(f'###### {glabel[gain]} ######'))\n",
- "\n",
- " gs = gridspec.GridSpec(2, 2)\n",
- " fig = plt.figure(figsize=(24, 32))\n",
- "\n",
- " axis = OrderedDict()\n",
- " axis = {\"ax0\": {\"cname\": \"{}\" ,\"gs\": gs[0, :], \"shrink\": 0.9, \"pad\": 0.05, \"label\": \"ADUs\", \"title\": '{}'},\n",
- " \"ax1\": {\"cname\": \"d-{}\",\"gs\": gs[1, 0], \"shrink\": 0.6, \"pad\": 0.1, \"label\": \"ADUs\", \"title\": 'Difference with previous {}'},\n",
- " \"ax2\": {\"cname\": \"dpct-{}\", \"gs\": gs[1, 1], \"shrink\": 0.6, \"pad\": 0.1, \"label\": \"%\", \"title\": 'Difference with previous {} %'}}\n",
- "\n",
- " for ax, axv in axis.items():\n",
- " # Add the min and max plot values for each axis.\n",
- " vmin, vmax = get_range(Mod_data[axv[\"cname\"].format(const_name)][gain_names[gain]], 2)\n",
- " ax = fig.add_subplot(axv[\"gs\"])\n",
- " geom.plot_data_fast(Mod_data[axv[\"cname\"].format(const_name)][gain_names[gain]],\n",
- " vmin=vmin, vmax=vmax, ax=ax, \n",
- " colorbar={'shrink': axv[\"shrink\"],\n",
- " 'pad': axv[\"pad\"]\n",
- " }\n",
- " )\n",
- "\n",
- " colorbar = ax.images[0].colorbar\n",
- " colorbar.set_label(axv[\"label\"])\n",
- "\n",
- " ax.set_title(axv[\"title\"].format(f\"{const_name} {glabel[gain]}\"), fontsize=15)\n",
- " ax.set_xlabel('Columns', fontsize=15)\n",
- " ax.set_ylabel('Rows', fontsize=15)\n",
+ " display(Markdown(f'###### {glabel} ######'))\n",
+ "\n",
+ " plot_const_and_delta(stacked_const, stacked_delta, const_name, glabel)\n",
"\n",
" plt.show()"
]
@@ -449,48 +489,55 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "scrolled": false
+ },
"outputs": [],
"source": [
"# Loop over modules and constants\n",
"for const_name, const in constants.items():\n",
+ " if const_name == 'BadPixelsDark':\n",
+ " continue # Displayed separately below\n",
+ " \n",
" display(Markdown(f'### Summary across Modules - {const_name}'))\n",
"\n",
" for gain in range(gainstages):\n",
" if const_name == 'ThresholdsDark':\n",
" if gain == 2:\n",
" continue\n",
- " glabel = threshold_names\n",
+ " glabel = threshold_names[gain]\n",
" else:\n",
- " glabel = gain_names\n",
+ " glabel = gain_names[gain]\n",
"\n",
- " if len(const.shape) == 5:\n",
- " data = np.copy(const[:, :, :, :, gain])\n",
+ " if const.ndim == 5:\n",
+ " data = const[:, :, :, :, gain]\n",
" else:\n",
- " data = np.copy(const[:, :, :, :])\n",
- " if const_name != 'BadPixelsDark':\n",
- " if \"BadPixelsDark\" in constants.keys():\n",
- " label = f'{const_name} value [ADU], good pixels only'\n",
- " if len(const.shape) == 5:\n",
- " data[constants['BadPixelsDark'][:, :, :, :, gain] > 0] = np.nan\n",
- " else:\n",
- " data[constants['BadPixelsDark'][:, :, :, :] > 0] = np.nan\n",
+ " data = const\n",
+ "\n",
+ " # Bad pixels are per gain stage, and you need thresholds to pick the gain\n",
+ " # stage, so we don't mask the thresholds.\n",
+ " if ('BadPixelsDark' in constants) and (const_name != 'ThresholdsDark'):\n",
+ " label = f'{const_name} value [ADU], good pixels only'\n",
+ " if const.ndim == 5:\n",
+ " goodpix = constants['BadPixelsDark'][:, :, :, :, gain] == 0\n",
" else:\n",
- " label = f'{const_name} value [ADU], good and bad pixels'\n",
- " datamean = np.nanmean(data, axis=(1, 2))\n",
- "\n",
- " fig = plt.figure(figsize=(15, 6), tight_layout={\n",
- " 'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})\n",
- " ax = fig.add_subplot(121)\n",
+ " goodpix = constants['BadPixelsDark'] == 0\n",
" else:\n",
- " label = 'Fraction of bad pixels'\n",
- " data[data > 0] = 1.0\n",
- " datamean = np.nanmean(data, axis=(1, 2))\n",
- " datamean[datamean == 1.0] = np.nan\n",
- "\n",
- " fig = plt.figure(figsize=(15, 6), tight_layout={\n",
- " 'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})\n",
- " ax = fig.add_subplot(111)\n",
+ " label = f'{const_name} value [ADU], good and bad pixels'\n",
+ " goodpix = [True] * data.shape[0]\n",
+ "\n",
+ " # Reduce data in parallel (one worker per module):\n",
+ " datamean = psh_ctx.alloc(data.shape[:1] + data.shape[3:], data.dtype)\n",
+ " datastd = psh_ctx.alloc(data.shape[:1] + data.shape[3:], data.dtype)\n",
+ " \n",
+ " def average_mem_cells(wid, i, _):\n",
+ " datamean[i] = np.mean(data[i], axis=(0, 1), where=goodpix[i])\n",
+ " datastd[i] = np.std(data[i], axis=(0, 1), where=goodpix[i])\n",
+ " psh_ctx.map(average_mem_cells, range(data.shape[0]))\n",
+ "\n",
+ " fig = plt.figure(figsize=(15, 6), tight_layout={\n",
+ " 'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})\n",
+ " ax = fig.add_subplot(121)\n",
"\n",
" d = []\n",
" for im, mod in enumerate(datamean):\n",
@@ -504,32 +551,80 @@
" x_label='Memory Cell ID',\n",
" y_label=label,\n",
" use_axis=ax,\n",
- " title='{}'.format(glabel[gain]),\n",
+ " title=glabel,\n",
+ " title_position=[0.5, 1.18],\n",
+ " legend='outside-top-ncol6-frame', legend_size='18%',\n",
+ " legend_pad=0.00)\n",
+ "\n",
+ " # Plot standard deviation\n",
+ " ax = fig.add_subplot(122)\n",
+ " if \"BadPixelsDark\" in constants.keys():\n",
+ " label = f'$\\sigma$ {const_name} [ADU], good pixels only'\n",
+ " else:\n",
+ " label = f'$\\sigma$ {const_name} [ADU], good and bad pixels'\n",
+ " d = []\n",
+ " for im, mod in enumerate(datastd):\n",
+ " d.append({'x': np.arange(mod.shape[0]),\n",
+ " 'y': mod,\n",
+ " 'drawstyle': 'steps-pre',\n",
+ " 'label': mod_names[im],\n",
+ " })\n",
+ "\n",
+ " _ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),\n",
+ " x_label='Memory Cell ID',\n",
+ " y_label=label,\n",
+ " use_axis=ax,\n",
+ " title=f'{glabel} $\\sigma$',\n",
+ " title_position=[0.5, 1.18],\n",
+ " legend='outside-top-ncol6-frame', legend_size='18%',\n",
+ " legend_pad=0.00)\n",
+ "\n",
+ " plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if 'BadPixelsDark' in constants:\n",
+ " display(Markdown(f'### Summary across Modules - BadPixelsDark'))\n",
+ "\n",
+ " bad_px_dark = constants['BadPixelsDark']\n",
+ "\n",
+ " for gain in range(gainstages):\n",
+ " glabel = gain_names[gain]\n",
+ "\n",
+ " if bad_px_dark.ndim == 5:\n",
+ " data = bad_px_dark[:, :, :, :, gain]\n",
+ " else:\n",
+ " data = bad_px_dark[:, :, :, :]\n",
+ "\n",
+ " bad_px_per_cell = np.count_nonzero(data, axis=(1, 2))\n",
+ " fraction_bad_per_cell = bad_px_per_cell / (data.shape[1] * data.shape[2])\n",
+ " fraction_bad_per_cell[fraction_bad_per_cell == 1.0] = np.nan\n",
+ "\n",
+ " fig = plt.figure(figsize=(15, 6), tight_layout={\n",
+ " 'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})\n",
+ " ax = fig.add_subplot(1, 1, 1)\n",
+ "\n",
+ " d = []\n",
+ " for im, mod in enumerate(fraction_bad_per_cell):\n",
+ " d.append({'x': np.arange(mod.shape[0]),\n",
+ " 'y': mod,\n",
+ " 'drawstyle': 'steps-pre',\n",
+ " 'label': mod_names[im],\n",
+ " })\n",
+ "\n",
+ " _ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),\n",
+ " x_label='Memory Cell ID',\n",
+ " y_label='Fraction of bad pixels',\n",
+ " use_axis=ax,\n",
+ " title=glabel,\n",
" title_position=[0.5, 1.18],\n",
" legend='outside-top-ncol6-frame', legend_size='18%',\n",
" legend_pad=0.00)\n",
- " if const_name != 'BadPixelsDark':\n",
- " ax = fig.add_subplot(122)\n",
- " if \"BadPixelsDark\" in constants.keys():\n",
- " label = f'$\\sigma$ {const_name} [ADU], good pixels only'\n",
- " else:\n",
- " label = f'$\\sigma$ {const_name} [ADU], good and bad pixels'\n",
- " d = []\n",
- " for im, mod in enumerate(np.nanstd(data, axis=(1, 2))):\n",
- " d.append({'x': np.arange(mod.shape[0]),\n",
- " 'y': mod,\n",
- " 'drawstyle': 'steps-pre',\n",
- " 'label': mod_names[im],\n",
- " })\n",
- "\n",
- " _ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),\n",
- " x_label='Memory Cell ID',\n",
- " y_label=label,\n",
- " use_axis=ax,\n",
- " title='{} $\\sigma$'.format(glabel[gain]),\n",
- " title_position=[0.5, 1.18],\n",
- " legend='outside-top-ncol6-frame', legend_size='18%',\n",
- " legend_pad=0.00)\n",
"\n",
" plt.show()"
]
@@ -565,58 +660,91 @@
"head = ['Module', 'High gain', 'Medium gain', 'Low gain']\n",
"head_th = ['Module', 'HG_MG threshold', 'MG_LG threshold']\n",
"for const_name, const in constants.items():\n",
+ " if const_name == 'BadPixelsDark':\n",
+ " continue # Handled below\n",
+ "\n",
+ " if const.ndim == 4: # Add gain dimension if not present\n",
+ " const = const[:, :, : , :, np.newaxis]\n",
+ " \n",
+ " if ('BadPixelsDark' in constants) and (const_name != 'ThresholdsDark'):\n",
+ " goodpix = constants['BadPixelsDark'] == 0\n",
+ " if goodpix.ndim == 4:\n",
+ " goodpix = goodpix[:, :, : , :, np.newaxis]\n",
+ " \n",
+ " label = f'### Average {const_name} [ADU], good pixels only'\n",
+ " else:\n",
+ " goodpix = [True] * const.shape[0]\n",
+ " label = f'### Average {const_name} [ADU], good and bad pixels'\n",
+ "\n",
+ " # Reduce data in parallel (one worker per module)\n",
+ " mean_by_module_gain = psh.alloc(const.shape[:1] + const.shape[4:], const.dtype)\n",
+ " std_by_module_gain = psh.alloc(const.shape[:1] + const.shape[4:], const.dtype)\n",
+ " \n",
+ " def average_module(wid, i, _):\n",
+ " mean_by_module_gain[i] = np.mean(const[i], axis=(0, 1, 2), where=goodpix[i])\n",
+ " std_by_module_gain[i] = np.std (const[i], axis=(0, 1, 2), where=goodpix[i])\n",
+ " psh_ctx.map(average_module, range(const.shape[0]))\n",
+ "\n",
+ " table = []\n",
+ "\n",
+ " for i_mod, mod in enumerate(mod_names):\n",
+ " t_line = [mod]\n",
+ " for gain in range(gainstages):\n",
+ " if const_name == 'ThresholdsDark' and gain == 2:\n",
+ " continue\n",
+ "\n",
+ " datamean = mean_by_module_gain[i_mod, gain]\n",
+ " datastd = std_by_module_gain[i_mod, gain]\n",
+ " t_line.append(f'{datamean:6.1f} $\\\\pm$ {datastd:6.1f}')\n",
+ "\n",
+ " table.append(t_line)\n",
+ "\n",
+ " display(Markdown(label))\n",
+ " header = head_th if const_name == 'ThresholdsDark' else head\n",
+ " md = display(Latex(tabulate.tabulate(\n",
+ " table, tablefmt='latex', headers=header)))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Bad pixels summary table\n",
+ "if 'BadPixelsDark' in constants: \n",
+ " bad_px_dark = constants['BadPixelsDark']\n",
+ "\n",
" table = []\n",
"\n",
" for i_mod, mod in enumerate(mod_names):\n",
"\n",
" t_line = [mod]\n",
" for gain in range(gainstages):\n",
- " \n",
- " if const_name == 'ThresholdsDark': \n",
- " if gain == 2:\n",
- " continue\n",
- " header = head_th\n",
- " else:\n",
- " header = head\n",
- " if len(const.shape) == 5: \n",
- " data = np.copy(const[i_mod, :, :, :, gain])\n",
+ " if bad_px_dark.ndim == 5: \n",
+ " data = bad_px_dark[i_mod, :, :, :, gain]\n",
" else:\n",
- " data = np.copy(const[i_mod, :, :, :])\n",
- " if const_name == 'BadPixelsDark':\n",
- " data[data > 0] = 1.0\n",
- "\n",
- " datasum = np.nansum(data)\n",
- " datamean = np.nanmean(data)\n",
- " if datamean == 1.0:\n",
- " datamean = np.nan\n",
- " datasum = np.nan\n",
- "\n",
- " t_line.append(f'{datasum:6.0f} ({datamean:6.3f}) ')\n",
- " label = '## Number(fraction) of bad pixels'\n",
- " else:\n",
- " if \"BadPixelsDark\" in constants.keys():\n",
- " data[constants['BadPixelsDark']\n",
- " [i_mod, :, :, :, gain] > 0] = np.nan\n",
- " label = f'### Average {const_name} [ADU], good pixels only'\n",
- " else:\n",
- " label = f'### Average {const_name} [ADU], good and bad pixels'\n",
+ " data = bad_px_dark[i_mod]\n",
"\n",
- " t_line.append(f'{np.nanmean(data):6.1f} $\\\\pm$ {np.nanstd(data):6.1f}')\n",
- " label = f'## Average {const_name} [ADU], good pixels only'\n",
+ " datasum = np.count_nonzero(data)\n",
+ " datamean = datasum / data.size\n",
+ "\n",
+ " t_line.append(f'{datasum:6.0f} ({datamean:6.3f}) ')\n",
+ " label = '## Number(fraction) of bad pixels'\n",
"\n",
" table.append(t_line)\n",
"\n",
" display(Markdown(label))\n",
" md = display(Latex(tabulate.tabulate(\n",
- " table, tablefmt='latex', headers=header)))"
+ " table, tablefmt='latex', headers=head)))"
]
}
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Offline Cal",
"language": "python",
- "name": "python3"
+ "name": "offline-cal"
},
"language_info": {
"codemirror_mode": {
@@ -628,7 +756,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.12"
+ "version": "3.8.10"
}
},
"nbformat": 4,