From d017a276d24e8f096dc4322c3fa2d01783f3d864 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Wed, 30 Jun 2021 15:53:41 +0200
Subject: [PATCH] separate branch from ePix100
---
...Jungfrau_Gain_Correct_and_Verify_NBC.ipynb | 12 +-
.../Characterize_Darks_ePix100_NBC.ipynb | 147 ++++++++++--------
.../ePix100/Correction_ePix100_NBC.ipynb | 87 ++++-------
3 files changed, 115 insertions(+), 131 deletions(-)
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index 1bace1840..d11e51c2b 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -358,8 +358,8 @@
" # Select memory cells\n",
" if memory_cells > 1:\n",
" m[m>16] = 0\n",
- " offset_map_cell = offset_map[m,...]\n",
- " mask_cell = mask[m,...]\n",
+ " offset_map_cell = offset_map[m, ...]\n",
+ " mask_cell = mask[m, ...]\n",
" else:\n",
" offset_map_cell = offset_map\n",
" mask_cell = mask\n",
@@ -367,16 +367,16 @@
" # Offset correction\n",
" offset = np.choose(\n",
" g,\n",
- " (offset_map_cell[...,0],\n",
- " offset_map_cell[...,1],\n",
- " offset_map_cell[...,2]),\n",
+ " (offset_map_cell[..., 0],\n",
+ " offset_map_cell[..., 1],\n",
+ " offset_map_cell[..., 2]),\n",
" )\n",
" d -= offset\n",
"\n",
" # Gain correction\n",
" if relative_gain:\n",
" if memory_cells > 1:\n",
- " gain_map_cell = gain_map[m,...]\n",
+ " gain_map_cell = gain_map[m, ...]\n",
" else:\n",
" gain_map_cell = gain_map\n",
" cal = np.choose(\n",
diff --git a/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb b/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
index 23d3a3a91..89ea1f460 100644
--- a/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
@@ -30,7 +30,7 @@
"receiver_id = \"RECEIVER\" # inset for receiver devices\n",
"path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
"h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data/image/pixels' # path in the HDF5 file to images\n",
- "h5path_t = 'data.backTemp' # path to find temperature at\n",
+ "h5path_t = '/INSTRUMENT/{}/DET/{}:daqOutput/data/backTemp' # path to find temperature at\n",
"h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
"\n",
"use_dir_creation_date = True\n",
@@ -57,14 +57,14 @@
"import os\n",
"import warnings\n",
"\n",
- "import numpy as np\n",
- "from extra_data import H5File\n",
+ "warnings.filterwarnings('ignore')\n",
"\n",
- "import XFELDetAna.xfelprofiler as xprof\n",
- "from XFELDetAna import xfelpyanatools as xana\n",
- "from XFELDetAna import xfelpycaltools as xcal\n",
- "from XFELDetAna.plotting.util import prettyPlotting\n",
- "from XFELDetAna.util import env\n",
+ "import h5py\n",
+ "import matplotlib.pyplot as plt\n",
+ "from IPython.display import Latex, Markdown, display\n",
+ "\n",
+ "%matplotlib inline\n",
+ "import numpy as np\n",
"from cal_tools.tools import (\n",
" get_dir_creation_date,\n",
" get_pdu_from_db,\n",
@@ -73,27 +73,32 @@
" save_const_to_h5,\n",
" send_to_db,\n",
")\n",
- "from iCalibrationDB import Conditions, Constants"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "env.iprofile = cluster_profile\n",
+ "from iCalibrationDB import Conditions, Constants, Detectors, Versions\n",
+ "from iCalibrationDB.detectors import DetectorTypes\n",
+ "from XFELDetAna.util import env\n",
"\n",
- "warnings.filterwarnings('ignore')\n",
+ "env.iprofile = cluster_profile\n",
+ "from XFELDetAna import xfelpyanatools as xana\n",
+ "from XFELDetAna import xfelpycaltools as xcal\n",
+ "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
+ "from XFELDetAna.plotting.util import prettyPlotting\n",
"\n",
"prettyPlotting = True\n",
+ "import XFELDetAna.xfelprofiler as xprof\n",
"\n",
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
+ "from XFELDetAna.xfelreaders import ChunkReader\n",
"\n",
"h5path = h5path.format(karabo_id, receiver_id)\n",
"h5path_t = h5path_t.format(karabo_id, receiver_id)\n",
- "h5path_cntrl = h5path_cntrl.format(karabo_id)"
+ "h5path_cntrl = h5path_cntrl.format(karabo_id)\n",
+ "\n",
+ "def nImagesOrLimit(nImages, limit):\n",
+ " if limit == 0:\n",
+ " return nImages\n",
+ " else:\n",
+ " return min(nImages, limit)"
]
},
{
@@ -112,7 +117,6 @@
"y = 768 # columns of the xPix100\n",
"\n",
"ped_dir = os.path.join(in_folder, f\"r{run:04d}\")\n",
- "h5file = H5File(os.path.join(ped_dir, \"RAW-R0182-EPIX02-S00000.h5\"))\n",
"fp_name = path_template.format(run, karabo_da[0]).format(sequence)\n",
"filename = os.path.join(ped_dir, fp_name)\n",
"\n",
@@ -132,41 +136,44 @@
"outputs": [],
"source": [
"sensorSize = [x, y]\n",
- "chunk_size = 100 # Number of images to read per chunk\n",
+ "chunkSize = 100 #Number of images to read per chunk\n",
"\n",
- "# Sensor area will be analysed according to blocksize\n",
+ "#Sensor area will be analysed according to blocksize\n",
"blockSize = [sensorSize[0] // 2, sensorSize[1] // 2]\n",
"xcal.defaultBlockSize = blockSize\n",
"cpuCores = 4 #Specifies the number of running cpu cores\n",
"memoryCells = 1 #No mamery cells\n",
"\n",
- "h5_data = f\"{karabo_id}/DET/{receiver_id}:daqOutput\"\n",
- "\n",
- "# Specifies total number of images to proceed\n",
- "n_imgs = h5file.get_data_counts(\n",
- " h5_data, \"data.image.pixels\").shape[0]\n",
- "\n",
- "print(\"Number of dark images to analyze: \", n_imgs)\n",
- "\n",
+ "#Specifies total number of images to proceed\n",
+ "nImages = fastccdreaderh5.getDataSize(filename, h5path)[0]\n",
+ "nImages = nImagesOrLimit(nImages, number_dark_frames)\n",
+ "print(\"\\nNumber of dark images to analyze: \", nImages)\n",
"run_parallel = False\n",
"\n",
- "integration_time = int(h5file.get_array(\n",
- " f\"{karabo_id}/DET/CONTROL\",\n",
- " \"expTime.value\")[0])\n",
- "temperature = np.mean(h5file.get_array(\n",
- " f\"{karabo_id}/DET/{receiver_id}:daqOutput\",\n",
- " f\"{h5path_t}\").values) / 100.\n",
- "\n",
- "if fix_temperature != 0:\n",
- " temperature_k = fix_temperature\n",
- "else:\n",
+ "with h5py.File(filename, 'r') as f:\n",
+ " integration_time = int(f[os.path.join(h5path_cntrl, \"CONTROL\",\"expTime\", \"value\")][0])\n",
+ " temperature = np.mean(f[h5path_t])/100.\n",
" temperature_k = temperature + 273.15\n",
- " print(\"Temperature is fixed!\")\n",
- "\n",
- "print(f\"Bias voltage is {bias_voltage} V\")\n",
- "print(f\"Detector integration time is set to {integration_time}\")\n",
- "print(f\"Mean temperature was {temperature:0.2f} °C / {temperature_k:0.2f} K\")\n",
- "print(f\"Operated in vacuum: {in_vacuum} \")"
+ " if fix_temperature != 0:\n",
+ " temperature_k = fix_temperature\n",
+ " print(\"Temperature is fixed!\")\n",
+ " print(f\"Bias voltage is {bias_voltage} V\")\n",
+ " print(f\"Detector integration time is set to {integration_time}\")\n",
+ " print(f\"Mean temperature was {temperature:0.2f} °C / {temperature_k:0.2f} K\")\n",
+ " print(f\"Operated in vacuum: {in_vacuum} \")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "reader = ChunkReader(filename, fastccdreaderh5.readData,\n",
+ " nImages, chunkSize,\n",
+ " path=h5path,\n",
+ " pixels_x=sensorSize[0],\n",
+ " pixels_y=sensorSize[1], )"
]
},
{
@@ -190,32 +197,18 @@
"metadata": {},
"outputs": [],
"source": [
- "for i in range(-(n_imgs // -chunk_size)):\n",
- " lst_img = chunk_size * (i + 1)\n",
- " if lst_img > n_imgs:\n",
- " lst_img = n_imgs\n",
- " \n",
- " data = h5file.get_array(\n",
- " h5_data, \"data.image.pixels\")[chunk_size*i:lst_img]\n",
- " data = np.moveaxis(data.values, 0, 2) \n",
+ "for data in reader.readChunks():\n",
" dx = np.count_nonzero(data, axis=(0, 1))\n",
" data = data[:, :, dx != 0]\n",
" histCalRaw.fill(data)\n",
- " noiseCal.fill(data) # Fill calculators with data"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
+ " noiseCal.fill(data) #Fill calculators with data\n",
+ " \n",
"constant_maps = {}\n",
- "constant_maps['Offset'] = noiseCal.getOffset() # Produce offset map\n",
- "constant_maps['Noise'] = noiseCal.get() # Produce noise map\n",
+ "constant_maps['Offset'] = noiseCal.getOffset() #Produce offset map\n",
+ "constant_maps['Noise'] = noiseCal.get() #Produce noise map\n",
"\n",
- "noiseCal.reset() # Reset noise calculator\n",
- "print(\"Initial constant maps are created\")"
+ "noiseCal.reset() #Reset noise calculator\n",
+ "print(\"Initial maps were created\")"
]
},
{
@@ -271,13 +264,18 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "scrolled": false
+ },
"outputs": [],
"source": [
"# Save constants to DB\n",
"dclass=\"ePix100\"\n",
"md = None\n",
"\n",
+ "# If PDU(db_module) is not given with input parameters \n",
+ "# retrieve the connected physical detector unit\n",
+ "\n",
"for const_name in constant_maps.keys():\n",
" det = getattr(Constants, dclass)\n",
" const = getattr(det, const_name)()\n",
@@ -293,6 +291,10 @@
" parm.lower_deviation = temp_limits\n",
" parm.upper_deviation = temp_limits\n",
"\n",
+ " # This should be used in case of running notebook \n",
+ " # by a different method other than myMDC which already\n",
+ " # sends CalCat info.\n",
+ " # TODO: Set db_module to \"\" by default in the first cell\n",
" if not db_module:\n",
" db_module = get_pdu_from_db(karabo_id, karabo_da, const,\n",
" condition, cal_db_interface,\n",
@@ -315,6 +317,13 @@
" f\"• Temperature: {temperature_k}\\n• In Vacuum: {in_vacuum}\\n\"\n",
" f\"• Creation time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
diff --git a/notebooks/ePix100/Correction_ePix100_NBC.ipynb b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
index de5062fbe..0c46331ff 100644
--- a/notebooks/ePix100/Correction_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
@@ -29,8 +29,8 @@
"receiver_id = \"RECEIVER\" # inset for receiver devices\n",
"path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
"h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data/image' # path in the HDF5 file to images\n",
- "h5path_t = 'data.backTemp' # path to find temperature at # TODO: modify in calibration_configurations.\n",
- "h5path_cntrl = '/CONTROL/{}/DET' # path to control data # TODO: remove from calibration_configurations.\n",
+ "h5path_t = '/INSTRUMENT/{}/DET/{}:daqOutput/data/backTemp' # path to find temperature at\n",
+ "h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
"\n",
"use_dir_creation_date = True # date constants injected before directory creation time\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8025\" # calibration DB interface to use\n",
@@ -75,7 +75,6 @@
"import h5py\n",
"import numpy as np\n",
"from IPython.display import Latex, display\n",
- "from extra_data import RunDirectory\n",
"from pathlib import Path\n",
"\n",
"import XFELDetAna.xfelprofiler as xprof\n",
@@ -113,6 +112,8 @@
"pattern_classification = False # do clustering.\n",
"\n",
"h5path = h5path.format(karabo_id, receiver_id)\n",
+ "h5path_t = h5path_t.format(karabo_id, receiver_id)\n",
+ "h5path_cntrl = h5path_cntrl.format(karabo_id)\n",
"plot_unit = 'ADU'\n",
"\n",
"if relative_gain:\n",
@@ -132,7 +133,6 @@
"\n",
"in_folder = Path(in_folder)\n",
"ped_dir = in_folder / f\"r{run:04d}\"\n",
- "run_dir = RunDirectory(ped_dir)\n",
"fp_name = path_template.format(run, karabo_da)\n",
"\n",
"print(f\"Reading from: {ped_dir / fp_name}\")\n",
@@ -162,37 +162,23 @@
"run_parallel = True\n",
"\n",
"# Read slow data from the first available sequence file.\n",
-<<<<<<< HEAD
- "filename = ped_dir / fp_name.format(sequences[0] if sequences[0] != -1 else 0)\n",
- "\n",
- "integration_time = int(run_dir.get_array(\n",
- " f\"{karabo_id}/DET/CONTROL\",\n",
- " \"expTime.value\")[0])\n",
- "temperature = np.mean(run_dir.get_array(\n",
- " f\"{karabo_id}/DET/{receiver_id}:daqOutput\",\n",
- " f\"{h5path_t}\").values) / 100.\n",
- "\n",
- "if fix_temperature != 0:\n",
- " temperature_k = fix_temperature\n",
- " print(\"Temperature is fixed!\")\n",
- "else:\n",
-=======
"filename = ped_dir / fp_name.format(\n",
" sequences[0] if sequences[0] != -1 else 0)\n",
"with h5py.File(filename, 'r') as f:\n",
" integration_time = int(\n",
" f[f\"{h5path_cntrl}/CONTROL/expTime/value\"][0])\n",
" temperature = np.mean(f[h5path_t]) / 100.\n",
->>>>>>> master
" temperature_k = temperature + 273.15\n",
- "\n",
- "print(f\"Bias voltage is {bias_voltage} V\")\n",
- "print(f\"Detector integration time is set to {integration_time}\")\n",
- "print(\n",
- " f\"Mean temperature was {temperature:0.2f} °C \"\n",
- " f\"/ {temperature_k:0.2f} K at beginning of the run.\"\n",
- ")\n",
- "print(f\"Operated in vacuum: {in_vacuum} \")\n",
+ " if fix_temperature != 0:\n",
+ " temperature_k = fix_temperature\n",
+ " print(\"Temperature is fixed!\")\n",
+ " print(f\"Bias voltage is {bias_voltage} V\")\n",
+ " print(f\"Detector integration time is set to {integration_time}\")\n",
+ " print(\n",
+ " f\"Mean temperature was {temperature:0.2f} °C \"\n",
+ " f\"/ {temperature_k:0.2f} K at beginning of run\"\n",
+ " )\n",
+ " print(f\"Operated in vacuum: {in_vacuum} \")\n",
"\n",
"out_folder = Path(out_folder)\n",
"if out_folder.is_dir() and not overwrite:\n",
@@ -486,24 +472,23 @@
" outfile: h5py,\n",
" h5base: str\n",
"):\n",
- " with h5py.File(f, \"r\") as infile:\n",
- " \"\"\" Copy and sanitize data in `infile`\n",
- " that is not touched by `correctEPIX`. \"\"\"\n",
- "\n",
- " if h5base.startswith(\"/\"):\n",
- " h5base = h5base[1:]\n",
- " dont_copy = [h5base+\"/pixels\"]\n",
- "\n",
- " def visitor(k, item):\n",
- " if k not in dont_copy:\n",
- " if isinstance(item, h5py.Group):\n",
- " outfile.create_group(k)\n",
- " elif isinstance(item, h5py.Dataset):\n",
- " group = str(k).split(\"/\")\n",
- " group = \"/\".join(group[:-1])\n",
- " infile.copy(k, outfile[group])\n",
- "\n",
- " infile.visititems(visitor)"
+ " \"\"\" Copy and sanitize data in `infile`\n",
+ " that is not touched by `correctEPIX`. \"\"\"\n",
+ "\n",
+ " if h5base.startswith(\"/\"):\n",
+ " h5base = h5base[1:]\n",
+ " dont_copy = [h5base+\"/pixels\"]\n",
+ "\n",
+ " def visitor(k, item):\n",
+ " if k not in dont_copy:\n",
+ " if isinstance(item, h5py.Group):\n",
+ " outfile.create_group(k)\n",
+ " elif isinstance(item, h5py.Dataset):\n",
+ " group = str(k).split(\"/\")\n",
+ " group = \"/\".join(group[:-1])\n",
+ " infile.copy(k, outfile[group])\n",
+ "\n",
+ " infile.visititems(visitor)"
]
},
{
@@ -515,22 +500,12 @@
"for f in seq_files:\n",
" data = None\n",
" out_file = out_folder / f.name.replace(\"RAW\", \"CORR\")\n",
-<<<<<<< HEAD
- " with h5py.File(out_file, \"w\") as ofile:\n",
- " try:\n",
- " copy_and_sanitize_non_cal_data(f, ofile, h5path)\n",
- " data = run_dir.get_array(\n",
- " f\"{karabo_id}/DET/{receiver_id}:daqOutput\",\n",
- " \"data.image.pixels\").values\n",
- " data = np.compress(np.any(data > 0, axis=(1, 2)), data, axis=0)\n",
-=======
" with h5py.File(f, \"r\") as infile, h5py.File(out_file, \"w\") as ofile: # noqa\n",
" try:\n",
" copy_and_sanitize_non_cal_data(infile, ofile, h5path)\n",
" data = infile[h5path+\"/pixels\"][()]\n",
" data = np.compress(\n",
" np.any(data > 0, axis=(1, 2)), data, axis=0)\n",
->>>>>>> master
" if limit_images > 0:\n",
" data = data[:limit_images, ...]\n",
"\n",
--
GitLab