From f27bae9022ef99fbeb972d98fa6c8cb465e50892 Mon Sep 17 00:00:00 2001 From: Karim Ahmed <karim.ahmed@xfel.eu> Date: Fri, 1 Oct 2021 19:17:34 +0200 Subject: [PATCH] [JUNGFRAU] Style changes for Dark and Correct notebooks --- ...Jungfrau_Gain_Correct_and_Verify_NBC.ipynb | 212 ++++++++++++------ ...rk_analysis_all_gains_burst_mode_NBC.ipynb | 60 ++--- 2 files changed, 176 insertions(+), 96 deletions(-) diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb index c386874f8..b62e4e1d5 100644 --- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb +++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb @@ -166,8 +166,9 @@ " else:\n", " table.append((fi, \"\", k, f))\n", " fi += 1\n", - " md = display(Latex(tabulate.tabulate(table, tablefmt='latex',\n", - " headers=[\"#\", \"module\", \"# module\", \"file\"])))\n", + " md = display(Latex(tabulate.tabulate(\n", + " table, tablefmt='latex',\n", + " headers=[\"#\", \"module\", \"# module\", \"file\"])))\n", "\n", "# restore the queue\n", "mapped_files, _, total_sequences, _, _ = map_modules_from_folder(\n", @@ -341,7 +342,14 @@ "outputs": [], "source": [ "# Correct a chunk of images for offset and gain\n", - "def correct_chunk(offset_map, mask, gain_map, memory_cells, relative_gain, inp):\n", + "def correct_chunk(\n", + " offset_map,\n", + " mask,\n", + " gain_map,\n", + " memory_cells,\n", + " relative_gain,\n", + " inp,\n", + "):\n", " fim_data = None\n", " gim_data = None\n", " rim_data = None\n", @@ -352,46 +360,61 @@ " d, g, m, ind, copy_sample = inp\n", " g[g==3] = 2\n", "\n", - " if copy_sample and ind==0:\n", - " if memory_cells==1:\n", + " if copy_sample and ind == 0:\n", + " if memory_cells == 1:\n", " rim_data = np.squeeze(copy.copy(d))\n", " else:\n", - " rim_data = np.squeeze(copy.copy(d[:,0,...]))\n", + " rim_data = np.squeeze(copy.copy(d[:, 0, ...]))\n", "\n", " # Select memory cells\n", - " if memory_cells>1:\n", + " if memory_cells > 1:\n", " m[m>16] = 0\n", - " offset_map_cell = offset_map[m,...]\n", - " mask_cell = mask[m,...]\n", + " offset_map_cell = offset_map[m, ...]\n", + " mask_cell = mask[m, ...]\n", " else:\n", " offset_map_cell = offset_map\n", " mask_cell = mask\n", "\n", " # Offset correction\n", - " offset = np.choose(g, (offset_map_cell[...,0], offset_map_cell[...,1], offset_map_cell[...,2]))\n", + " offset = np.choose(\n", + " g,\n", + " (offset_map_cell[..., 0],\n", + " offset_map_cell[..., 1],\n", + " offset_map_cell[..., 2]),\n", + " )\n", " d -= offset\n", "\n", " # Gain correction\n", " if relative_gain:\n", - " if memory_cells>1:\n", - " gain_map_cell = gain_map[m,...]\n", + " if memory_cells > 1:\n", + " gain_map_cell = gain_map[m, ...]\n", " else:\n", " gain_map_cell = gain_map\n", - " cal = np.choose(g, (gain_map_cell[..., 0], gain_map_cell[..., 1], gain_map_cell[..., 2]))\n", + " cal = np.choose(\n", + " g,\n", + " (gain_map_cell[..., 0],\n", + " gain_map_cell[..., 1],\n", + " gain_map_cell[..., 2]),\n", + " )\n", " d /= cal \n", "\n", - " msk = np.choose(g, (mask_cell[...,0], mask_cell[...,1], mask_cell[...,2]))\n", + " msk = np.choose(\n", + " g,\n", + " (mask_cell[..., 0],\n", + " mask_cell[..., 1],\n", + " mask_cell[..., 2]),\n", + " )\n", "\n", " # Store sample of data for plotting\n", - " if copy_sample and ind==0:\n", - " if memory_cells==1:\n", + " if copy_sample and ind == 0:\n", + " if memory_cells == 1:\n", " fim_data = np.squeeze(copy.copy(d))\n", " gim_data = np.squeeze(copy.copy(g))\n", " msk_data = np.squeeze(copy.copy(msk))\n", " else:\n", - " fim_data = np.squeeze(copy.copy(d[:,1,...]))\n", - " gim_data = np.squeeze(copy.copy(g[:,1,...]))\n", - " msk_data = np.squeeze(copy.copy(msk[:,1,...]))\n", + " fim_data = np.squeeze(copy.copy(d[:, 1, ...]))\n", + " gim_data = np.squeeze(copy.copy(g[:, 1, ...]))\n", + " msk_data = np.squeeze(copy.copy(msk[:, 1, ...]))\n", "\n", " except Exception as e:\n", " err = e\n", @@ -435,16 +458,20 @@ " # Chunk always contains >= 1 complete image\n", " chunk_shape = (chunk_size_idim, 1) + oshape[-2:]\n", "\n", - " ddset = outfile.create_dataset(h5path_f+\"/adc\",\n", - " oshape,\n", - " chunks=chunk_shape,\n", - " dtype=np.float32)\n", - "\n", - " mskset = outfile.create_dataset(h5path_f+\"/mask\",\n", - " oshape,\n", - " chunks=chunk_shape,\n", - " dtype=np.uint32,\n", - " compression=\"gzip\", compression_opts=1, shuffle=True)\n", + " ddset = outfile.create_dataset(\n", + " h5path_f+\"/adc\",\n", + " oshape,\n", + " chunks=chunk_shape,\n", + " dtype=np.float32)\n", + "\n", + " mskset = outfile.create_dataset(\n", + " h5path_f+\"/mask\",\n", + " oshape,\n", + " chunks=chunk_shape,\n", + " dtype=np.uint32,\n", + " compression=\"gzip\",\n", + " compression_opts=1,\n", + " shuffle=True)\n", " # Parallelize over chunks of images\n", " inp = []\n", " max_ind = oshape[0]\n", @@ -452,15 +479,15 @@ "\n", " # If chunk size is not given maximum 12+1 chunks is expected\n", " if chunk_size == 0:\n", - " chunk_size = max_ind//12\n", + " chunk_size = max_ind // 12\n", " print(f'Chunk size: {chunk_size}')\n", "\n", " ts = time.time()\n", " while ind<max_ind:\n", - " d = infile[h5path_f+\"/adc\"][ind:ind+chunk_size,...].astype(np.float32)\n", - " g = infile[h5path_f+\"/gain\"][ind:ind+chunk_size,...]\n", + " d = infile[h5path_f+\"/adc\"][ind:ind+chunk_size, ...].astype(np.float32)\n", + " g = infile[h5path_f+\"/gain\"][ind:ind+chunk_size, ...]\n", " if h5path_f+\"/memoryCell\" in infile:\n", - " m = infile[h5path_f+\"/memoryCell\"][ind:ind+chunk_size,...]\n", + " m = infile[h5path_f+\"/memoryCell\"][ind:ind+chunk_size, ...]\n", " else:\n", " m = None\n", " print(f'To process: {d.shape}')\n", @@ -471,7 +498,15 @@ " ts = time.time()\n", "\n", " print(f'Run {len(inp)} processes')\n", - " p = partial(correct_chunk, offset_map, mask, gain_map, memory_cells, relative_gain)\n", + "\n", + " p = partial(\n", + " correct_chunk,\n", + " offset_map,\n", + " mask,\n", + " gain_map,\n", + " memory_cells,\n", + " relative_gain,\n", + " )\n", "\n", " r = pool.map(p, inp)\n", " \n", @@ -484,10 +519,10 @@ " ts = time.time()\n", "\n", " for rr in r:\n", - " ind, cdata, cmask, _,_,_,_, err = rr\n", + " ind, cdata, cmask, _, _, _, _, err = rr\n", " data_size = cdata.shape[0]\n", - " ddset[ind:ind+data_size,...] = cdata\n", - " mskset[ind:ind+data_size,...] = cmask\n", + " ddset[ind:ind+data_size, ...] = cdata\n", + " mskset[ind:ind+data_size, ...] = cmask\n", " if err != '':\n", " print(f'Error: {err}')\n", "\n", @@ -504,10 +539,21 @@ "outputs": [], "source": [ "def do_2d_plot(data, edges, y_axis, x_axis, title):\n", - " fig = plt.figure(figsize=(10,10))\n", + " fig = plt.figure(figsize=(10, 10))\n", " ax = fig.add_subplot(111)\n", - " extent = [np.min(edges[1]), np.max(edges[1]),np.min(edges[0]), np.max(edges[0])]\n", - " im = ax.imshow(data[::-1,:], extent=extent, aspect=\"auto\", norm=LogNorm(vmin=1, vmax=np.max(data)))\n", + " extent = [\n", + " np.min(edges[1]),\n", + " np.max(edges[1]),\n", + " np.min(edges[0]),\n", + " np.max(edges[0]),\n", + " ]\n", + "\n", + " im = ax.imshow(\n", + " data[::-1, :],\n", + " extent=extent,\n", + " aspect=\"auto\",\n", + " norm=LogNorm(vmin=1, vmax=np.max(data))\n", + " )\n", " ax.set_xlabel(x_axis)\n", " ax.set_ylabel(y_axis)\n", " ax.set_title(title)\n", @@ -522,11 +568,17 @@ "outputs": [], "source": [ "for mod in rim_data: \n", - " h, ex, ey = np.histogram2d(rim_data[mod].flatten(),\n", - " gim_data[mod].flatten(),\n", - " bins=[100, 4],\n", - " range=[[0, 10000], [0, 4]])\n", - " do_2d_plot(h, (ex, ey), \"Signal (ADU)\", \"Gain Bit Value\", f'Module {mod}')" + " h, ex, ey = np.histogram2d(\n", + " rim_data[mod].flatten(),\n", + " gim_data[mod].flatten(),\n", + " bins=[100, 4],\n", + " range=[[0, 10000], [0, 4]],\n", + " )\n", + " do_2d_plot(\n", + " h, (ex, ey),\n", + " \"Signal (ADU)\",\n", + " \"Gain Bit Value\",\n", + " f\"Module {mod}\")" ] }, { @@ -545,11 +597,15 @@ "outputs": [], "source": [ "for mod in rim_data: \n", - " fig = plt.figure(figsize=(20,10))\n", + " fig = plt.figure(figsize=(20, 10))\n", " ax = fig.add_subplot(111)\n", - " im = ax.imshow(np.mean(rim_data[mod],axis=0),\n", - " vmin=min(0.75*np.median(rim_data[mod][rim_data[mod] > 0]), 2000),\n", - " vmax=max(1.5*np.median(rim_data[mod][rim_data[mod] > 0]), 16000), cmap=\"jet\")\n", + "\n", + " im = ax.imshow(\n", + " np.mean(rim_data[mod],axis=0),\n", + " vmin=min(0.75*np.median(rim_data[mod][rim_data[mod] > 0]), 2000),\n", + " vmax=max(1.5*np.median(rim_data[mod][rim_data[mod] > 0]), 16000),\n", + " cmap=\"jet\")\n", + "\n", " ax.set_title(f'Module {mod}')\n", " cb = fig.colorbar(im, ax=ax)" ] @@ -570,11 +626,15 @@ "outputs": [], "source": [ "for mod in rim_data: \n", - " fig = plt.figure(figsize=(20,10))\n", + " fig = plt.figure(figsize=(20, 10))\n", " ax = fig.add_subplot(111)\n", - " im = ax.imshow(np.mean(fim_data[mod],axis=0),\n", - " vmin=min(0.75*np.median(fim_data[mod][fim_data[mod] > 0]), -0.5),\n", - " vmax=max(2.*np.median(fim_data[mod][fim_data[mod] > 0]), 100), cmap=\"jet\")\n", + "\n", + " im = ax.imshow(\n", + " np.mean(fim_data[mod],axis=0),\n", + " vmin=min(0.75*np.median(fim_data[mod][fim_data[mod] > 0]), -0.5),\n", + " vmax=max(2.*np.median(fim_data[mod][fim_data[mod] > 0]), 100),\n", + " cmap=\"jet\")\n", + "\n", " ax.set_title(f'Module {mod}', size=18)\n", " cb = fig.colorbar(im, ax=ax)" ] @@ -595,11 +655,15 @@ "outputs": [], "source": [ "for mod in rim_data: \n", - " fig = plt.figure(figsize=(20,10))\n", + " fig = plt.figure(figsize=(20, 10))\n", " ax = fig.add_subplot(111)\n", - " im = ax.imshow(fim_data[mod][0,...],\n", - " vmin=min(0.75*np.median(fim_data[mod][0,...]), -0.5),\n", - " vmax=max(2.*np.median(fim_data[mod][0,...]), 100), cmap=\"jet\")\n", + "\n", + " im = ax.imshow(\n", + " fim_data[mod][0, ...],\n", + " vmin=min(0.75*np.median(fim_data[mod][0, ...]), -0.5),\n", + " vmax=max(2.*np.median(fim_data[mod][0, ...]), 100),\n", + " cmap=\"jet\")\n", + "\n", " ax.set_title(f'Module {mod}', size=18)\n", " cb = fig.colorbar(im, ax=ax)" ] @@ -620,13 +684,23 @@ "for mod in rim_data: \n", " fig = plt.figure(figsize=(20,10))\n", " ax = fig.add_subplot(211)\n", - " h = ax.hist(fim_data[mod].flatten(), bins=1000, range=(-100, 1000), log=True)\n", + " h = ax.hist(\n", + " fim_data[mod].flatten(),\n", + " bins=1000,\n", + " range=(-100, 1000),\n", + " log=True,\n", + " )\n", " l = ax.set_xlabel(\"Signal (keV)\")\n", " l = ax.set_ylabel(\"Counts\")\n", " _ = ax.set_title(f'Module {mod}')\n", "\n", " ax = fig.add_subplot(212)\n", - " h = ax.hist(fim_data[mod].flatten(), bins=1000, range=(-1000, 10000), log=True)\n", + " h = ax.hist(\n", + " fim_data[mod].flatten(),\n", + " bins=1000,\n", + " range=(-1000, 10000),\n", + " log=True,\n", + " )\n", " l = ax.set_xlabel(\"Signal (keV)\")\n", " l = ax.set_ylabel(\"Counts\")\n", " _ = ax.set_title(f'Module {mod}')" @@ -648,10 +722,11 @@ "outputs": [], "source": [ "for mod in rim_data: \n", - " fig = plt.figure(figsize=(20,10))\n", + " fig = plt.figure(figsize=(20, 10))\n", " ax = fig.add_subplot(111)\n", - " im = ax.imshow(np.max(gim_data[mod], axis=0), vmin=0,\n", - " vmax=3, cmap=\"jet\")\n", + " im = ax.imshow(\n", + " np.max(gim_data[mod], axis=0),\n", + " vmin=0, vmax=3, cmap=\"jet\")\n", " ax.set_title(f'Module {mod}', size=18)\n", " cb = fig.colorbar(im, ax=ax)" ] @@ -672,8 +747,11 @@ "source": [ "table = []\n", "for item in BadPixels:\n", - " table.append((item.name, f\"{item.value:016b}\"))\n", - "md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=[\"Bad pixel type\", \"Bit mask\"])))" + " table.append(\n", + " (item.name, f\"{item.value:016b}\"))\n", + "md = display(Latex(tabulate.tabulate(\n", + " table, tablefmt='latex',\n", + " headers=[\"Bad pixel type\", \"Bit mask\"])))" ] }, { @@ -692,9 +770,11 @@ "outputs": [], "source": [ "for mod in rim_data: \n", - " fig = plt.figure(figsize=(20,10))\n", + " fig = plt.figure(figsize=(20, 10))\n", " ax = fig.add_subplot(111)\n", - " im = ax.imshow(np.log2(msk_data[mod][0,...]), vmin=0, vmax=32, cmap=\"jet\")\n", + " im = ax.imshow(\n", + " np.log2(msk_data[mod][0,...]),\n", + " vmin=0, vmax=32, cmap=\"jet\")\n", " ax.set_title(f'Module {mod}', size=18)\n", " cb = fig.colorbar(im, ax=ax)" ] diff --git a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb index 922d58f8c..cd604ac09 100644 --- a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb +++ b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb @@ -35,11 +35,11 @@ "h5path_cntrl = '/CONTROL/{}/DET/{}' # path to control data\n", "karabo_da_control = \"JNGFRCTRL00\" # file inset for control data\n", "\n", - "use_dir_creation_date = True # use dir creation date\n", + "use_dir_creation_date = True # use dir creation date\n", "cal_db_interface = 'tcp://max-exfl016:8016' # calibrate db interface to connect to\n", "cal_db_timeout = 300000 # timeout on caldb requests\n", - "local_output = True # output constants locally\n", - "db_output = False # output constants to database\n", + "local_output = True # output constants locally\n", + "db_output = False # output constants to database\n", "\n", "integration_time = 1000 # integration time in us, will be overwritten by value in file\n", "gain_setting = 0 # 0 for dynamic, forceswitchg1, forceswitchg2, 1 for dynamichg0, fixedgain1, fixgain2. Will be overwritten by value in file\n", @@ -52,7 +52,7 @@ "memoryCells = 16 # number of memory cells\n", "db_module = \"\" # ID of module in calibration database, this parameter is ignored in the notebook. TODO: remove from calibration_configurations.\n", "manual_slow_data = False # if true, use manually entered bias_voltage and integration_time values\n", - "time_limits = 0.025 # to find calibration constants later on, the integration time is allowed to vary by 0.5 us\n", + "time_limits = 0.025 # to find calibration constants later on, the integration time is allowed to vary by 0.5 us\n", "operation_mode = '' # Detector operation mode, optional" ] }, @@ -125,7 +125,9 @@ "xRange = [0, 0+sensorSize[0]]\n", "yRange = [0, 0+sensorSize[1]]\n", "gains = [0, 1, 2]\n", + "\n", "h5path = h5path.format(karabo_id, receiver_id)\n", + "\n", "creation_time = None\n", "if use_dir_creation_date:\n", " creation_time = get_dir_creation_date(in_folder, run_high)\n", @@ -198,16 +200,12 @@ "\n", " myRange = range(0, n_files)\n", " control_path = h5path_cntrl.format(karabo_id_control, receiver_control_id)\n", - " \n", + "\n", " this_run_mcells, sc_start = check_memoryCells(fp_path.format(0).format(myRange[0]), control_path)\n", - " \n", + "\n", " if mod not in noise_map:\n", " if not manual_slow_data:\n", - " with h5py.File(fp_path.format(0), 'r') as f:\n", " run_path = h5path_run.format(karabo_id_control, receiver_control_id)\n", - " integration_time = float(f[f'{run_path}/exposureTime/value'][()]*1e6)\n", - " bias_voltage = int(np.squeeze(f[f'{run_path}/vHighVoltage/value'])[0])\n", - " \n", " if r_n == run_high:\n", " try:\n", " gain_s = f[f'/RUN/{karabo_id_control}/DET/CONTROL/settings/value'][0].decode()\n", @@ -241,7 +239,7 @@ " print(\"Reading data from {}\".format(fp_path))\n", " print(\"Run is: {}\".format(r_n))\n", " print(\"HDF5 path: {}\".format(h5path))\n", - " \n", + "\n", " imageRange = [0, filep_size*len(myRange)]\n", " reader = JFChunkReader(filename = fp_path, readFun = jfreader.readData, size = filep_size, chunkSize = chunkSize,\n", " path = h5path, image_range=imageRange, pixels_x = sensorSize[0], pixels_y = sensorSize[1],\n", @@ -249,7 +247,7 @@ " memoryCells=this_run_mcells, blockSize=blockSize)\n", "\n", " for data in reader.readChunks():\n", - " \n", + "\n", " images = np.array(data[0], dtype=np.float)\n", " gainmaps = np.array(data[1], dtype=np.uint16)\n", " trainId = np.array(data[2])\n", @@ -257,31 +255,32 @@ " acelltable = np.array(data[4])\n", " n_tr += acelltable.shape[-1]\n", " this_tr = acelltable.shape[-1]\n", - " \n", - " \n", - " \n", + "\n", " idxs = np.nonzero(trainId)[0]\n", " images = images[..., idxs]\n", " gainmaps = gainmaps[..., idxs]\n", " fr_num = fr_num[..., idxs]\n", " acelltable = acelltable[..., idxs]\n", - " \n", + "\n", " if memoryCells == 1:\n", " acelltable -= sc_start\n", "\n", " n_empty_trains += this_tr - acelltable.shape[-1]\n", " n_empty_sc += len(acelltable[acelltable > 15])\n", "\n", - " if gain > 0 and memoryCells == 16: ## throwing away all the SC entries except the first for lower gains\n", + " # throwing away all the SC entries except\n", + " # the first for lower gains.\n", + " if gain > 0 and memoryCells == 16: \n", " acelltable[1:] = 255\n", - " \n", + "\n", " # makes 4-dim vecs into 3-dim\n", " # makes 2-dim into 1-dim\n", " # leaves 1-dim and 3-dim vecs\n", "\n", - " images, gainmaps, acelltable = rollout_data([images, gainmaps, acelltable]) \n", - " \n", - " images, gainmaps, acelltable = sanitize_data_cellid([images, gainmaps], acelltable) # removes entries with cellID 255\n", + " images, gainmaps, acelltable = rollout_data([images, gainmaps, acelltable])\n", + "\n", + " # removes entries with cellID 255\n", + " images, gainmaps, acelltable = sanitize_data_cellid([images, gainmaps], acelltable)\n", " valid_data.append(images)\n", " valid_cellids.append(acelltable)\n", "\n", @@ -294,10 +293,10 @@ " noise_map[mod][..., cell, gain] = np.std(thiscell, axis=2)\n", " offset_map[mod][..., cell, gain] = np.mean(thiscell, axis=2)\n", "\n", - "\n", " print(f'G{gain:01d} dark calibration')\n", - " print('Missed {:d} out of {:d} trains'.format(n_empty_trains, n_tr))\n", - " print('Lost {:d} images out of {:d}'.format(n_empty_sc, this_run_mcells * (n_tr - n_empty_trains)))" + " print(f'Missed {n_empty_trains:d} out of {n_tr:d} trains')\n", + " print(\n", + " f'Lost {n_empty_sc:d} images out of {this_run_mcells*(n_tr-n_empty_trains):d}') # noqa" ] }, { @@ -461,12 +460,13 @@ "for g_idx in gains:\n", " for cell in range(memoryCells):\n", " bad_pixels = bad_pixels_map[mod][:, :, cell, g_idx]\n", - " fn_0 = heatmapPlot(np.swapaxes(bad_pixels, 0, 1), \n", - " y_label=\"Row\",\n", - " x_label=\"Column\",\n", - " lut_label=f\"Badpixels {g_name[g_idx]} [ADCu]\",\n", - " aspect=1.,\n", - " vmin=0, title=f'G{g_idx} Bad pixel map - Cell {cell:02d} - Module {mod}')" + " fn_0 = heatmapPlot(\n", + " np.swapaxes(bad_pixels, 0, 1),\n", + " y_label=\"Row\",\n", + " x_label=\"Column\",\n", + " lut_label=f\"Badpixels {g_name[g_idx]} [ADCu]\",\n", + " aspect=1.,\n", + " vmin=0, title=f'G{g_idx} Bad pixel map - Cell {cell:02d} - Module {mod}')" ] }, { -- GitLab