From 4b9648bde0009920471cb3e0d21878729f262d8d Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Tue, 12 Oct 2021 18:36:32 +0200
Subject: [PATCH] use extra-data to find available files to correct
---
.../pnCCD/Characterize_pnCCD_Dark_NBC.ipynb | 6 +-
notebooks/pnCCD/Correct_pnCCD_NBC.ipynb | 67 +++++++++----------
2 files changed, 33 insertions(+), 40 deletions(-)
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
index 1b675b896..4fdc49927 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
@@ -144,8 +144,8 @@
"pixels_y = 1024 # number of columns of the pnCCD \n",
"print(f\"pnCCD size is: {pixels_x}x{pixels_y} pixels.\")\n",
"\n",
- "ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "instrument_src = instrument_source_template.format(karabo_id, receiver_id)\n",
+ "instrument_src = instrument_source_template.format(\n",
+ " karabo_id, receiver_id)\n",
"\n",
"# Output Folder Creation:\n",
"os.makedirs(out_folder, exist_ok=True)\n",
@@ -191,7 +191,7 @@
"source": [
"step_timer.start()\n",
"\n",
- "run_dc = RunDirectory(ped_dir)\n",
+ "run_dc = RunDirectory(f\"in_folder/r{run:04d}\")\n",
"\n",
"# extract control data\n",
"if karabo_da_control:\n",
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index f6fe7448c..b2495e302 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -87,7 +87,6 @@
"# Dont apply any corrections if only_offset is requested.\n",
"if not only_offset:\n",
" corr_bools[\"relgain\"] = relgain\n",
- " corr_bools[\"cti\"] = cti\n",
" corr_bools[\"common_mode\"] = common_mode\n",
" corr_bools[\"pattern_class\"] = pattern_classification"
]
@@ -143,10 +142,6 @@
"outputs": [],
"source": [
"# Calibration Database Settings, and Some Initial Run Parameters & Paths:\n",
- "\n",
- "if sequences[0] == -1:\n",
- " sequences = None\n",
- "\n",
"display(Markdown('### Initial Settings and Paths'))\n",
"pixels_x = 1024 # rows of pnCCD in pixels \n",
"pixels_y = 1024 # columns of pnCCD in pixels\n",
@@ -185,21 +180,7 @@
"metadata": {},
"outputs": [],
"source": [
- "# set everything up filewise\n",
- "mapped_files, _, total_sequences, _, _ = map_modules_from_folder(\n",
- " in_folder=in_folder,\n",
- " run=run,\n",
- " path_template=path_template,\n",
- " karabo_da=[karabo_da],\n",
- " sequences=sequences,\n",
- " qm_naming=False,\n",
- ")\n",
- "file_list = []\n",
- "print(f\"Processing a total of {total_sequences} sequence files:\")\n",
- "# TODO: use extra data and sort it.\n",
- "for f in mapped_files[karabo_da].queue:\n",
- " file_list.append(f)\n",
- " print(f)"
+ "step_timer = StepTimer()"
]
},
{
@@ -210,15 +191,11 @@
"source": [
"# extract control data\n",
"if karabo_da_control:\n",
+ " step_timer.start()\n",
" mdl_src = mdl_source_template.format(karabo_id, \"{}\")\n",
" ctrl_src = control_source_template.format(karabo_id)\n",
- " #TODO sequences and select files\n",
- " ctrl_dc = RunDirectory(Path(in_folder) / f\"r{run:04d}\").select(\n",
- " [\n",
- " [ctrl_src, \"*\"],\n",
- " [mdl_src.format(\"*\"), \"*\"],]\n",
- " )\n",
- " ctrl_data = pnccdlib.PnccdCtrl(ctrl_dc, ctrl_src, mdl_src)\n",
+ " run_dc = RunDirectory(Path(in_folder) / f\"r{run:04d}\")\n",
+ " ctrl_data = pnccdlib.PnccdCtrl(run_dc, ctrl_src, mdl_src)\n",
"\n",
" if bias_voltage == 0.:\n",
" bias_voltage = ctrl_data.get_bias_voltage()\n",
@@ -228,6 +205,7 @@
" fix_temperature_top = ctrl_data.get_fix_temperature_top()\n",
" if fix_temperature_bot == 0:\n",
" fix_temperature_bot = ctrl_data.get_fix_temperature_bot()\n",
+ " step_timer.done_step(\"Reading control parameters.\")\n",
"\n",
"# Printing the Parameters Read from the Data File:\n",
"display(Markdown('### Detector Parameters'))\n",
@@ -238,6 +216,23 @@
"print(f\"Bottom pnCCD sensor is at temperature of {fix_temperature_bot:0.2f} K\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "seq_files = []\n",
+ "for f in run_dc.select(f\"*{karabo_id}*\").files:\n",
+ " fpath = Path(f.filename)\n",
+ " if fpath.match(f\"*{karabo_da}*.h5\"):\n",
+ " seq_files.append(fpath)\n",
+ "if sequences != [-1]:\n",
+ " seq_files = sorted([f for f in seq_files if any(f.match(f\"*-S{s:05d}.h5\") for s in sequences)])\n",
+ "print(f\"Processing a total of {len(seq_files)} sequence files:\")\n",
+ "print(*seq_files, sep='\\n')"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -340,8 +335,6 @@
"metadata": {},
"outputs": [],
"source": [
- "step_timer = StepTimer()\n",
- "\n",
"# Output Folder Creation:\n",
"os.makedirs(out_folder, exist_ok=True if overwrite else False)"
]
@@ -733,24 +726,24 @@
"source": [
"# Data corrections and event classifications happen here.\n",
"# Also, the corrected data are written to datasets:\n",
- "for seq_n, seq_f in enumerate(file_list):\n",
+ "for seq_n, seq_f in enumerate(seq_files):\n",
" f_dc = H5File(seq_f)\n",
- " out_fileb = f\"{out_folder}/{seq_f.split('/')[-1]}\"\n",
- " out_file = out_fileb.replace(\"RAW\", \"CORR\")\n",
+ " out_file = f\"{out_folder}/{seq_f.name}\".replace(\"RAW\", \"CORR\")\n",
"\n",
" step_timer.start()\n",
+ "\n",
" dshape = f_dc[instrument_src, \"data.image\"].shape\n",
- " n_imgs = dshape[0]\n",
+ " n_trains = dshape[0]\n",
" # If you want to analyze only a certain number of frames\n",
" # instead of all available good frames.\n",
" if limit_images > 0:\n",
- " n_imgs = min(n_imgs, limit_images)\n",
- " data_shape = (n_imgs, dshape[1], dshape[2])\n",
+ " n_trains = min(n_trains, limit_images)\n",
+ " data_shape = (n_trains, dshape[1], dshape[2])\n",
" print(f\"Correcting file: {seq_f} of shape {data_shape}.\")\n",
"\n",
" data_dc = f_dc.select(\n",
" instrument_src, \"data.image\",\n",
- " require_all=True).select_trains(np.s_[:n_imgs])\n",
+ " require_all=True).select_trains(np.s_[:n_trains])\n",
" \n",
" # Allocating shared arrays for data arrays for each correction stage.\n",
"\n",
@@ -815,7 +808,7 @@
" # Storing corrected data sources.\n",
" with h5py.File(out_file, 'w') as ofile:\n",
" # Copy RAW non-calibrated sources.\n",
- " with h5py.File(f, 'r') as sfile:\n",
+ " with h5py.File(seq_f, 'r') as sfile:\n",
" h5_copy_except.h5_copy_except_paths(\n",
" sfile, ofile,\n",
" [\"INSTRUMENT/\"+instrument_src+\"/data/image\"],\n",
--
GitLab