From 78288a4cf9b26c3f116699218f8e0864c337987e Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Tue, 27 Apr 2021 14:39:05 +0200
Subject: [PATCH] MR suggestions
---
.../ePix100/Correction_ePix100_NBC.ipynb | 198 ++++++++----------
1 file changed, 88 insertions(+), 110 deletions(-)
diff --git a/notebooks/ePix100/Correction_ePix100_NBC.ipynb b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
index 952a3e7f3..bbfc6eae0 100644
--- a/notebooks/ePix100/Correction_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
@@ -100,7 +100,7 @@
"\n",
"warnings.filterwarnings('ignore')\n",
"\n",
- "prettyPlotting=True\n",
+ "prettyPlotting = True\n",
"\n",
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
@@ -115,9 +115,6 @@
"metadata": {},
"outputs": [],
"source": [
- "if sequences[0] == -1:\n",
- " sequences = None\n",
- "\n",
"h5path = h5path.format(karabo_id, receiver_id)\n",
"h5path_t = h5path_t.format(karabo_id, receiver_id)\n",
"h5path_cntrl = h5path_cntrl.format(karabo_id)\n",
@@ -137,12 +134,12 @@
"source": [
"x = 708 # rows of the ePix100\n",
"y = 768 # columns of the ePix100\n",
- " \n",
- "ped_dir = Path(f\"{in_folder}/r{run:04d}\")\n",
+ "\n",
+ "in_folder = Path(in_folder)\n",
+ "ped_dir = in_folder / f\"r{run:04d}\"\n",
"fp_name = path_template.format(run, karabo_da)\n",
- "fp_path = ped_dir / fp_name\n",
"\n",
- "print(f\"Reading from: {fp_path}\")\n",
+ "print(f\"Reading from: {ped_dir / fp_name}\")\n",
"print(f\"Run is: {run}\")\n",
"print(f\"HDF5 path: {h5path}\")\n",
"print(f\"Data is output to: {out_folder}\")\n",
@@ -170,7 +167,7 @@
"run_parallel = True\n",
"\n",
"# Read slow data from the first available sequence file.\n",
- "filename = str(fp_path).format(sequences[0] if sequences else 0)\n",
+ "filename = ped_dir / fp_name.format(sequences[0] if sequences[0]!=-1 else 0)\n",
"with h5py.File(filename, 'r') as f:\n",
" integration_time = int(f[f\"{h5path_cntrl}/CONTROL/expTime/value\"][0])\n",
" temperature = np.mean(f[h5path_t])/100.\n",
@@ -195,28 +192,15 @@
"metadata": {},
"outputs": [],
"source": [
- "dirlist = sorted(ped_dir.iterdir())\n",
- "file_list = []\n",
- "total_sequences = 0\n",
- "fsequences = []\n",
- "for entry in dirlist:\n",
- "\n",
- " # only h5 file\n",
- " abs_entry = ped_dir / entry\n",
- " if abs_entry.is_file() and abs_entry.suffix == \".h5\":\n",
- "\n",
- " if sequences is None:\n",
- " sequences_list = range(len(dirlist))\n",
- " else:\n",
- " sequences_list = sequences\n",
- " for seq in sequences_list:\n",
- " if path_template.format(run, karabo_da).format(seq) in str(abs_entry):\n",
- " file_list.append(abs_entry)\n",
- " total_sequences += 1\n",
- " fsequences.append(seq)\n",
- "\n",
- "sequences = fsequences\n",
- "print(f\"Processing a total of {total_sequences} sequence files\")"
+ "# Glob the right *.h5 fast data files.\n",
+ "seq_files = sorted(ped_dir.glob(f\"*{karabo_da}*.h5\"))\n",
+ "\n",
+ "# If a set of sequences requested to correct,\n",
+ "# adapt seq_files list.\n",
+ "if sequences != [-1]:\n",
+ " seq_files = [f for f in seq_files if any(f.match(f\"*-S{s:05d}.h5\") for s in sequences)]\n",
+ "\n",
+ "print(f\"Processing a total of {len(seq_files)} sequence files\")"
]
},
{
@@ -226,7 +210,7 @@
"outputs": [],
"source": [
"# Table of sequence files to process\n",
- "table = [(k, f) for k, f in enumerate(file_list)]\n",
+ "table = [(k, f) for k, f in enumerate(seq_files)]\n",
"\n",
"if len(table): \n",
" md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=[\"#\", \"file\"]))) "
@@ -320,13 +304,13 @@
"\n",
"if relative_gain:\n",
" gainCorrection = xcal.RelativeGainCorrection(\n",
- " sensorSize, \n",
- " 1./const_data[\"RelativeGain\"][..., None],\n",
- " nCells=memoryCells,\n",
- " parallel=run_parallel,\n",
- " cores=cpuCores,\n",
- " blockSize=blockSize,\n",
- " gains=None,\n",
+ " sensorSize, \n",
+ " 1./const_data[\"RelativeGain\"][..., None],\n",
+ " nCells=memoryCells,\n",
+ " parallel=run_parallel,\n",
+ " cores=cpuCores,\n",
+ " blockSize=blockSize,\n",
+ " gains=None,\n",
" )"
]
},
@@ -381,8 +365,8 @@
" [x, y],\n",
" commonModeBlockSize,\n",
" commonModeAxisR,\n",
- " nCells = memoryCells,\n",
- " noiseMap = const_data[\"Noise\"],\n",
+ " nCells=memoryCells,\n",
+ " noiseMap=const_data[\"Noise\"],\n",
" runParallel=run_parallel,\n",
" stats=True,\n",
" )\n",
@@ -403,10 +387,10 @@
" split_evt_primary_threshold, \n",
" split_evt_secondary_threshold,\n",
" split_evt_mip_threshold,\n",
- " tagFirstSingles = 0, \n",
+ " tagFirstSingles=0, \n",
" nCells=memoryCells, \n",
" cores=cpuCores, \n",
- " allowElongated = False,\n",
+ " allowElongated=False,\n",
" blockSize=[x, y],\n",
" runParallel=run_parallel,\n",
")\n",
@@ -451,9 +435,7 @@
" \n",
" if h5base.startswith(\"/\"):\n",
" h5base = h5base[1:]\n",
- " dont_copy = ['pixels']\n",
- " dont_copy = [h5base+f\"/{do}\"\n",
- " for do in dont_copy]\n",
+ " dont_copy = [h5base+\"/pixels\"]\n",
"\n",
" def visitor(k, item):\n",
" if k not in dont_copy:\n",
@@ -473,76 +455,72 @@
"metadata": {},
"outputs": [],
"source": [
- "for f in file_list:\n",
+ "for f in seq_files:\n",
" data = None\n",
" out_file = out_folder / f.name.replace(\"RAW\", \"CORR\")\n",
- " with h5py.File(f, 'r') as infile:\n",
- " with h5py.File(out_file, \"w\") as ofile:\n",
- " try:\n",
- " copy_and_sanitize_non_cal_data(infile, ofile, h5path)\n",
- " data = infile[h5path+\"/pixels\"][()]\n",
- " data = np.compress(np.any(data>0, axis=(1,2)), data, axis=0)\n",
- " if limit_images > 0:\n",
- " data = data[:limit_images,...]\n",
- "\n",
- " oshape = data.shape\n",
- " data = np.moveaxis(data, 0, 2)\n",
- " ddset = ofile.create_dataset(\n",
- " h5path+\"/pixels\",\n",
+ " with h5py.File(f, \"r\") as infile, h5py.File(out_file, \"w\") as ofile:\n",
+ " try:\n",
+ " copy_and_sanitize_non_cal_data(infile, ofile, h5path)\n",
+ " data = infile[h5path+\"/pixels\"][()]\n",
+ " data = np.compress(np.any(data>0, axis=(1,2)), data, axis=0)\n",
+ " if limit_images > 0:\n",
+ " data = data[:limit_images,...]\n",
+ "\n",
+ " oshape = data.shape\n",
+ " data = np.moveaxis(data, 0, 2)\n",
+ " ddset = ofile.create_dataset(\n",
+ " h5path+\"/pixels\",\n",
+ " oshape,\n",
+ " chunks=(chunk_size_idim, oshape[1], oshape[2]),\n",
+ " dtype=np.float32)\n",
+ "\n",
+ " # Offset correction.\n",
+ " data = offsetCorrection.correct(data.astype(np.float32))\n",
+ "\n",
+ " # relative gain correction.\n",
+ " if relative_gain:\n",
+ " data = gainCorrection.correct(data.astype(np.float32))\n",
+ " if photon_energy > 0:\n",
+ " data /= photon_energy\n",
+ "\n",
+ " histCalOffsetCor.fill(data)\n",
+ " ddset[...] = np.moveaxis(data, 2, 0)\n",
+ "\n",
+ " # Common mode correction\n",
+ " if common_mode:\n",
+ " ddsetcm = ofile.create_dataset(\n",
+ " h5path+\"/pixels_cm\",\n",
" oshape,\n",
" chunks=(chunk_size_idim, oshape[1], oshape[2]),\n",
" dtype=np.float32)\n",
"\n",
- " # Offset correction.\n",
- " data = offsetCorrection.correct(data.astype(np.float32))\n",
- " # relative gain correction.\n",
- " if relative_gain:\n",
- " data = gainCorrection.correct(data.astype(np.float32))\n",
- " if photon_energy > 0:\n",
- " data /= photon_energy\n",
- "\n",
- " histCalOffsetCor.fill(data)\n",
- " ddset[...] = np.moveaxis(data, 2, 0)\n",
- " except Exception as e:\n",
- " print(f\"Couldn't calibrate data in {f}: {e}\")\n",
- "\n",
- " if common_mode:\n",
- " with h5py.File(out_file, \"a\") as ofiled:\n",
- " try:\n",
- " # copy_and_sanitize_non_cal_data(infile, ofiled, h5path)\n",
- " ddsetcm = ofiled.create_dataset(\n",
- " h5path+\"/pixels_cm\",\n",
- " oshape,\n",
- " chunks=(chunk_size_idim, oshape[1], oshape[2]),\n",
- " dtype=np.float32)\n",
- "\n",
- " ddsetc = ofiled.create_dataset(\n",
- " h5path+\"/pixels_classified\",\n",
- " oshape,\n",
- " chunks=(chunk_size_idim, oshape[1], oshape[2]),\n",
- " dtype=np.float32, compression=\"gzip\")\n",
- "\n",
- " ddsetp = ofiled.create_dataset(\n",
- " h5path+\"/patterns\",\n",
- " oshape,\n",
- " chunks=(chunk_size_idim, oshape[1], oshape[2]),\n",
- " dtype=np.int32, compression=\"gzip\")\n",
- "\n",
- " # row common mode correction.\n",
- " data = cmCorrection.correct(data) \n",
- " histCalCMCor.fill(data)\n",
- " ddsetcm[...] = np.moveaxis(data, 2, 0)\n",
- "\n",
- " data, patterns = patternClassifier.classify(data)\n",
- "\n",
- " data[data < (split_evt_primary_threshold*const_data[\"Noise\"])] = 0\n",
- " ddsetc[...] = np.moveaxis(data, 2, 0)\n",
- " ddsetp[...] = np.moveaxis(patterns, 2, 0)\n",
- "\n",
- " data[patterns != 100] = np.nan\n",
- " histCalSECor.fill(data)\n",
- " except Exception as e:\n",
- " print(f\"ERROR applying common mode correction for {f}: {e}\")"
+ " ddsetc = ofile.create_dataset(\n",
+ " h5path+\"/pixels_classified\",\n",
+ " oshape,\n",
+ " chunks=(chunk_size_idim, oshape[1], oshape[2]),\n",
+ " dtype=np.float32, compression=\"gzip\")\n",
+ "\n",
+ " ddsetp = ofile.create_dataset(\n",
+ " h5path+\"/patterns\",\n",
+ " oshape,\n",
+ " chunks=(chunk_size_idim, oshape[1], oshape[2]),\n",
+ " dtype=np.int32, compression=\"gzip\")\n",
+ "\n",
+ " # row common mode correction.\n",
+ " data = cmCorrection.correct(data) \n",
+ " histCalCMCor.fill(data)\n",
+ " ddsetcm[...] = np.moveaxis(data, 2, 0)\n",
+ "\n",
+ " data, patterns = patternClassifier.classify(data)\n",
+ "\n",
+ " data[data < (split_evt_primary_threshold*const_data[\"Noise\"])] = 0\n",
+ " ddsetc[...] = np.moveaxis(data, 2, 0)\n",
+ " ddsetp[...] = np.moveaxis(patterns, 2, 0)\n",
+ "\n",
+ " data[patterns != 100] = np.nan\n",
+ " histCalSECor.fill(data)\n",
+ " except Exception as e:\n",
+ " print(f\"ERROR applying common mode correction for {f}: {e}\")"
]
},
{
--
GitLab