diff --git a/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb b/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..f648d5655603d46f17f2dba045d7ef31a142ab46
--- /dev/null
+++ b/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
@@ -0,0 +1,399 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# ePix100 Dark Characterization\n",
+ "\n",
+ "Author: M. Karnevskiy, Version 1.0\n",
+ "\n",
+ "The following notebook provides dark image analysis of the ePix100 detector.\n",
+ "\n",
+ "Dark characterization evaluates offset and noise of the detector and gives information about bad pixels. Resulting maps are saved as .h5 files for a latter use and injected to calibration DB."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "in_folder = '/gpfs/exfel/d/raw/MID/201921/p002419' # input folder, required\n",
+ "out_folder = '/gpfs/exfel/data/scratch/karnem/test/' # output folder, required\n",
+ "path_template = 'RAW-R{:04d}-DA01-S{{:05d}}.h5' # the template to use to access data\n",
+ "run = 94 # which run to read data from, required\n",
+ "number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
+ "cluster_profile = \"noDB\" # ipcluster profile to use\n",
+ "h5path = '/INSTRUMENT/MID_EXP_EPIX-1/DET/RECEIVER:daqOutput/data/image/pixels' # path in the HDF5 file to images\n",
+ "h5path_t = '/INSTRUMENT/MID_EXP_EPIX-1/DET/RECEIVER:daqOutput/data/backTemp' # path to find temperature at\n",
+ "h5path_cntrl = '/CONTROL/MID_EXP_EPIX-1/DET' # path to control data\n",
+ "cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
+ "local_output = False # output also in as H5 files\n",
+ "temp_limits = 5 # limit for parameter Operational temperature\n",
+ "sequence = 0 # sequence file to use\n",
+ "use_dir_creation_date = False\n",
+ "detector_db = 'ePix100_M15' # detector instance\n",
+ "bias_voltage = 200 # bias voltage\n",
+ "in_vacuum = False # detector operated in vacuum "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Disabled GPU usage after pyCuda import failed!: libcuda.so.1: cannot open shared object file: No such file or directory\n",
+ "Using Cython were available\n"
+ ]
+ }
+ ],
+ "source": [
+ "import XFELDetAna.xfelprofiler as xprof\n",
+ "\n",
+ "profiler = xprof.Profiler()\n",
+ "profiler.disable()\n",
+ "from XFELDetAna.util import env\n",
+ "env.iprofile = cluster_profile\n",
+ "\n",
+ "import warnings\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "from XFELDetAna import xfelpycaltools as xcal\n",
+ "from XFELDetAna import xfelpyanatools as xana\n",
+ "from XFELDetAna.plotting.util import prettyPlotting\n",
+ "\n",
+ "prettyPlotting = True\n",
+ "from XFELDetAna.xfelreaders import ChunkReader\n",
+ "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
+ "from cal_tools.tools import get_dir_creation_date\n",
+ "\n",
+ "from iCalibrationDB import (ConstantMetaData, Constants, Conditions, Detectors,\n",
+ " Versions)\n",
+ "from iCalibrationDB.detectors import DetectorTypes\n",
+ "\n",
+ "import numpy as np\n",
+ "import h5py\n",
+ "import matplotlib.pyplot as plt\n",
+ "% matplotlib inline\n",
+ "\n",
+ "def nImagesOrLimit(nImages, limit):\n",
+ " if limit == 0:\n",
+ " return nImages\n",
+ " else:\n",
+ " return min(nImages, limit)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Reading data from: /home/karnem/myscratch/dataTest/ePix/r0094/RAW-R0094-DA01-S00000.h5\n",
+ "\n",
+ "Run number: 94\n",
+ "HDF5 path: /INSTRUMENT/MID_EXP_EPIX-1/DET/RECEIVER:daqOutput/data/image/pixels\n",
+ "Using 2019-03-28T13:42:56.588021 as creation time\n"
+ ]
+ }
+ ],
+ "source": [
+ "x = 708 # rows of the xPix100\n",
+ "y = 768 # columns of the xPix100\n",
+ "\n",
+ "ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
+ "fp_name = path_template.format(run).format(sequence)\n",
+ "filename = '{}/{}'.format(ped_dir, fp_name)\n",
+ "\n",
+ "print(\"Reading data from: {}\\n\".format(filename))\n",
+ "print(\"Run number: {}\".format(run))\n",
+ "print(\"HDF5 path: {}\".format(h5path))\n",
+ "if use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run)\n",
+ " print(\"Using {} as creation time\".format(creation_time.isoformat()))\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "Number of dark images to analyze: 500\n",
+ "Bias voltage is 200 V\n",
+ "Detector integration time is set to 5.0\n",
+ "Mean temperature was 14.16 °C / 287.31 K\n",
+ "Operated in vacuum: False \n"
+ ]
+ }
+ ],
+ "source": [
+ "sensorSize = [x, y]\n",
+ "chunkSize = 100 #Number of images to read per chunk\n",
+ "\n",
+ "#Sensor area will be analysed according to blocksize\n",
+ "blockSize = [sensorSize[0] // 2, sensorSize[1] // 2]\n",
+ "xcal.defaultBlockSize = blockSize\n",
+ "cpuCores = 4 #Specifies the number of running cpu cores\n",
+ "memoryCells = 1 #No mamery cells\n",
+ "\n",
+ "#Specifies total number of images to proceed\n",
+ "nImages = fastccdreaderh5.getDataSize(filename, h5path)[0]\n",
+ "nImages = nImagesOrLimit(nImages, number_dark_frames)\n",
+ "print(\"\\nNumber of dark images to analyze: \", nImages)\n",
+ "run_parallel = False\n",
+ "\n",
+ "with h5py.File(filename, 'r') as f:\n",
+ " integration_time = int(f['{}/CONTROL/expTime/value'.format(h5path_cntrl)][0])\n",
+ " temperature = np.mean(f[h5path_t])/100.\n",
+ " temperature_k = temperature + 273.15\n",
+ "\n",
+ " print(\"Bias voltage is {} V\".format(bias_voltage))\n",
+ " print(\"Detector integration time is set to {}\".format(integration_time))\n",
+ " print(\"Mean temperature was {:0.2f} °C / {:0.2f} K\".format(temperature,\n",
+ " temperature_k))\n",
+ " \n",
+ " print(\"Operated in vacuum: {} \".format(in_vacuum))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "reader = ChunkReader(filename, fastccdreaderh5.readData,\n",
+ " nImages, chunkSize,\n",
+ " path=h5path,\n",
+ " pixels_x=sensorSize[0],\n",
+ " pixels_y=sensorSize[1], )\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "noiseCal = xcal.NoiseCalculator(sensorSize, memoryCells,\n",
+ " cores=cpuCores, blockSize=blockSize,\n",
+ " parallel=run_parallel)\n",
+ "histCalRaw = xcal.HistogramCalculator(sensorSize, bins=1000,\n",
+ " range=[0, 10000], parallel=False,\n",
+ " memoryCells=memoryCells,\n",
+ " cores=cpuCores, blockSize=blockSize)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Initial maps were created\n"
+ ]
+ }
+ ],
+ "source": [
+ "for data in reader.readChunks():\n",
+ " dx = np.count_nonzero(data, axis=(0, 1))\n",
+ " data = data[:, :, dx != 0]\n",
+ " histCalRaw.fill(data)\n",
+ " noiseCal.fill(data) #Fill calculators with data\n",
+ " \n",
+ "constant_maps = {}\n",
+ "constant_maps['Offset'] = noiseCal.getOffset() #Produce offset map\n",
+ "constant_maps['Noise'] = noiseCal.get() #Produce noise map\n",
+ "\n",
+ "noiseCal.reset() #Reset noise calculator\n",
+ "print(\"Initial maps were created\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "#**************OFFSET MAP HISTOGRAM***********#\n",
+ "ho, co = np.histogram(constant_maps['Offset'].flatten(), bins=700)\n",
+ "\n",
+ "do = {'x': co[:-1],\n",
+ " 'y': ho,\n",
+ " 'y_err': np.sqrt(ho[:]),\n",
+ " 'drawstyle': 'bars',\n",
+ " 'color': 'cornflowerblue',\n",
+ " }\n",
+ "\n",
+ "fig = xana.simplePlot(do, figsize='1col', aspect=2,\n",
+ " x_label='Offset (ADU)',\n",
+ " y_label=\"Counts\", y_log=True,\n",
+ " )\n",
+ "\n",
+ "#*****NOISE MAP HISTOGRAM FROM THE OFFSET CORRECTED DATA*******#\n",
+ "hn, cn = np.histogram(constant_maps['Noise'].flatten(), bins=200)\n",
+ "\n",
+ "dn = {'x': cn[:-1],\n",
+ " 'y': hn,\n",
+ " 'y_err': np.sqrt(hn[:]),\n",
+ " 'drawstyle': 'bars',\n",
+ " 'color': 'cornflowerblue',\n",
+ " }\n",
+ "\n",
+ "fig = xana.simplePlot(dn, figsize='1col', aspect=2,\n",
+ " x_label='Noise (ADU)',\n",
+ " y_label=\"Counts\",\n",
+ " y_log=True)\n",
+ "\n",
+ "#**************HEAT MAPS*******************#\n",
+ "fig = xana.heatmapPlot(constant_maps['Offset'][:, :, 0],\n",
+ " x_label='Columns', y_label='Rows',\n",
+ " lut_label='Offset (ADU)',\n",
+ " x_range=(0, y),\n",
+ " y_range=(0, x), vmin=1000, vmax=4000)\n",
+ "\n",
+ "fig = xana.heatmapPlot(constant_maps['Noise'][:, :, 0],\n",
+ " x_label='Columns', y_label='Rows',\n",
+ " lut_label='Noise (ADU)',\n",
+ " x_range=(0, y),\n",
+ " y_range=(0, x), vmax=2 * np.mean(constant_maps['Noise']))\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Inject Offset constants from 2019-03-28 13:42:55.885900\n",
+ "Inject Noise constants from 2019-03-28 13:42:55.885900\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Save constants to DB\n",
+ "dclass=\"EPix100\"\n",
+ "consts = [\"Offset\", \"Noise\"]\n",
+ "\n",
+ "for const_name in consts:\n",
+ " metadata = ConstantMetaData()\n",
+ " det = getattr(Constants, dclass)\n",
+ " const = getattr(det, const_name)()\n",
+ " const.data = constant_maps[const_name].data\n",
+ "\n",
+ " metadata.calibration_constant = const\n",
+ "\n",
+ " # set the operating condition\n",
+ " dcond = Conditions.Dark\n",
+ " condition = getattr(dcond, dclass)(bias_voltage=bias_voltage,\n",
+ " integration_time=integration_time,\n",
+ " temperature=temperature_k,\n",
+ " in_vacuum=in_vacuum)\n",
+ "\n",
+ " for parm in condition.parameters:\n",
+ " if parm.name == \"Sensor Temperature\":\n",
+ " parm.lower_deviation = temp_limits\n",
+ " parm.upper_deviation = temp_limits\n",
+ "\n",
+ " device = getattr(Detectors, detector_db)\n",
+ "\n",
+ " metadata.detector_condition = condition\n",
+ "\n",
+ " # specify the a version for this constant\n",
+ " if use_dir_creation_date:\n",
+ " metadata.calibration_constant_version = Versions.FromFile(device=device,\n",
+ " file_path= filename)\n",
+ " else:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ "\n",
+ " #metadata.send(cal_db_interface)\n",
+ " print(\"Inject {} constants from {}\".format(const_name, \n",
+ " metadata.calibration_constant_version.begin_at))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.4.3"
+ },
+ "latex_envs": {
+ "LaTeX_envs_menu_present": true,
+ "autocomplete": true,
+ "bibliofile": "biblio.bib",
+ "cite_by": "apalike",
+ "current_citInitial": 1,
+ "eqLabelWithNumbers": true,
+ "eqNumInitial": 1,
+ "hotkeys": {
+ "equation": "Ctrl-E",
+ "itemize": "Ctrl-I"
+ },
+ "labels_anchors": false,
+ "latex_user_defs": false,
+ "report_style_numbering": false,
+ "user_envs_cfg": false
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index fce778755b53a299a11097db9d54dab52496526f..bcb860f9509c58761d42cac781cc522d9875ab1c 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -135,5 +135,12 @@ notebooks = {
"cluster cores": 4},
},
},
-
+ "EPIX": {
+ "DARK": {
+ "notebook": "notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb",
+ "concurrency": {"parameter": None,
+ "default concurrency": None,
+ "cluster cores": 4},
+ },
+ },
}