diff --git a/notebooks/Jungfrau/Jungfrau_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Correct_and_Verify_NBC.ipynb
deleted file mode 100644
index e7941ec7cf1c64b0b2d0920c839a8440248c01a8..0000000000000000000000000000000000000000
--- a/notebooks/Jungfrau/Jungfrau_Correct_and_Verify_NBC.ipynb
+++ /dev/null
@@ -1,601 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Jungfrau Offline Correction #\n",
- "\n",
- "Author: European XFEL Detector Group, Version: 0.1\n",
- "\n",
- "Offline Calibration for the Jungfrau Detector"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "in_folder = \"/gpfs/exfel/exp/FXE/201801/p002118/raw/\" # the folder to read data from, required\n",
- "run = 2 # runs to process, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/haufs/test/\" # the folder to output to, required\n",
- "calfile = \"\" # path to calibration file. Leave empty if all data should come from DB, not actually used, makes automode happy\n",
- "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
- "mem_cells = 1 # memory cells in data, not actually used, makes automode happy\n",
- "overwrite = True # set to True if existing data should be overwritten\n",
- "no_relative_gain = True # do not do relative gain correction\n",
- "cluster_profile = \"noDB\"\n",
- "bias_voltage = 90 # will be overwritten by value in file\n",
- "cal_db_interface = \"tcp://max-exfl016:8018\" #\"tcp://max-exfl016:8015#8025\" # the database interface to use\n",
- "use_dir_creation_date = False # use the creation data of the input dir for database queries\n",
- "sequences_per_node = 10 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel\n",
- "photon_energy = 9.2 # photon energy in keV\n",
- "nodb = False # if set only file-based constants will be used\n",
- "chunk_size_idim = 1 # chunking size of imaging dimension, adjust if user software is sensitive to this.\n",
- "path_template = 'RAW-R{:04d}-DA06-S000{{:02d}}.h5' # template to use for file name, double escape sequence number\n",
- "integration_time = 1000 # integration time in us, will be overwritten by value in file\n",
- "h5path = '/INSTRUMENT/FXE_XAD_JF1M1/DET/RECEIVER:daqOutput/data' # path in H5 file under which images are located\n",
- "memcells = 1 \n",
- "\n",
- "def balance_sequences(in_folder, run, sequences, sequences_per_node):\n",
- " import glob\n",
- " import re\n",
- " import numpy as np\n",
- " if sequences_per_node != 0:\n",
- " sequence_files = glob.glob(\"{}/r{:04d}/*-S*.h5\".format(in_folder, run))\n",
- " seq_nums = set()\n",
- " for sf in sequence_files:\n",
- " seqnum = re.findall(r\".*-S([0-9]*).h5\", sf)[0]\n",
- " seq_nums.add(int(seqnum))\n",
- " seq_nums -= set(sequences)\n",
- " return [l.tolist() for l in np.array_split(list(seq_nums),\n",
- " len(seq_nums)//sequences_per_node+1)]\n",
- " else:\n",
- " return sequences\n",
- " "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import warnings\n",
- "warnings.filterwarnings('ignore')\n",
- "\n",
- "\n",
- "import matplotlib\n",
- "matplotlib.use('agg')\n",
- "import matplotlib.pyplot as plt\n",
- "%matplotlib inline\n",
- "import numpy as np\n",
- "import XFELDetAna.xfelpyanatools as xana\n",
- "import XFELDetAna.xfelpycaltools as xcal\n",
- "\n",
- "from cal_tools.enums import BadPixels\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "\n",
- "from cal_tools.cal_tools import (get_dir_creation_date)\n",
- "\n",
- "import os\n",
- "import h5py\n",
- "\n",
- "from XFELDetAna.util import env\n",
- "env.iprofile = cluster_profile\n",
- "\n",
- "from XFELDetAna.detectors.jungfrau import readerPSI as jfreaderPSI\n",
- "from XFELDetAna.detectors.jungfrau import reader as jfreader\n",
- "from XFELDetAna.detectors.jungfrau.jf_chunk_reader import JFChunkReader\n",
- "from XFELDetAna.detectors.jungfrau.util import non_empty_trains\n",
- "\n",
- "# so constants relevant for the analysis\n",
- "cpuCores = 1\n",
- "is_parallel = False\n",
- "sensorSize = [1024, 512]\n",
- "blockSize = [1024, 512]\n",
- "xRange = [0, 0+sensorSize[0]]\n",
- "yRange = [0, 0+sensorSize[1]]\n",
- "gains = [0, 1, 2]\n",
- "\n",
- "ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "\n",
- "if ped_dir[-1] == \"/\":\n",
- " ped_dir = ped_dir[:-1]\n",
- "out_folder = \"{}/{}\".format(out_folder, os.path.split(ped_dir)[-1])\n",
- "\n",
- "if not os.path.exists(out_folder):\n",
- " os.makedirs(out_folder)\n",
- "elif not overwrite:\n",
- " raise AttributeError(\"Output path exists! Exiting\") \n",
- "\n",
- "\n",
- "\n",
- "fp_name = path_template.format(run)\n",
- "fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
- "\n",
- "\n",
- "\n",
- "filep_size = 500\n",
- "myRange_P = sequences\n",
- "path = h5path\n",
- "\n",
- "if sequences[0] == -1:\n",
- " sequences = None\n",
- " \n",
- "print(\"Reading data from {}\".format(fp_path))\n",
- "print(\"Run is: {}\".format(run))\n",
- "print(\"HDF5 path: {}\".format(h5path))\n",
- "\n",
- "creation_time = None\n",
- "if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run)\n",
- " print(\"Using {} as creation time\".format(creation_time))\n",
- " \n",
- "cal_timeout = 600000 #ms"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import h5py\n",
- "with h5py.File(fp_path.format(0), 'r') as f:\n",
- " integration_time = int(f['/RUN/FXE_XAD_JF1M1/DET/CONTROL/exposureTime/value'][()]*1e6)\n",
- " bias_voltage = int(np.squeeze(f['/RUN/FXE_XAD_JF1M1/DET/CONTROL/vHighVoltage/value'])[0])\n",
- " print(\"Integration time is {} us\".format(integration_time))\n",
- " print(\"Bias voltage is {} V\".format(bias_voltage))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "dirlist = sorted(os.listdir(ped_dir))\n",
- "file_list = []\n",
- "total_sequences = 0\n",
- "fsequences = []\n",
- "for entry in dirlist:\n",
- " #only h5 file\n",
- " abs_entry = \"{}/{}\".format(ped_dir, entry)\n",
- " if os.path.isfile(abs_entry) and os.path.splitext(abs_entry)[1] == \".h5\":\n",
- " \n",
- " if sequences is None:\n",
- " for seq in range(len(dirlist)):\n",
- " \n",
- " if path_template.format(run).format(seq) in abs_entry:\n",
- " file_list.append(abs_entry)\n",
- " total_sequences += 1\n",
- " fsequences.append(seq)\n",
- " else:\n",
- " for seq in sequences:\n",
- " if path_template.format(run).format(seq) in abs_entry:\n",
- " file_list.append(os.path.abspath(abs_entry))\n",
- " total_sequences += 1\n",
- " fsequences.append(seq)\n",
- "sequences = fsequences"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import copy\n",
- "from IPython.display import HTML, display, Markdown, Latex\n",
- "import tabulate\n",
- "print(\"Processing a total of {} sequence files\".format(total_sequences))\n",
- "table = []\n",
- "\n",
- "\n",
- "for k, f in enumerate(file_list):\n",
- " table.append((k, f))\n",
- " \n",
- "md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=[\"#\", \"file\"]))) "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "## offset\n",
- "\n",
- "metadata = ConstantMetaData()\n",
- "offset = Constants.jungfrau.Offset()\n",
- "metadata.calibration_constant = offset\n",
- "\n",
- "# set the operating condition\n",
- "condition = Conditions.Dark.jungfrau(memory_cells=1, bias_voltage=bias_voltage,\n",
- " integration_time=integration_time)\n",
- "device = Detectors.jungfrau1\n",
- "\n",
- "\n",
- "metadata.detector_condition = condition\n",
- "\n",
- "# specify the a version for this constant\n",
- "metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "metadata.retrieve(cal_db_interface, when=creation_time, timeout=cal_timeout)\n",
- "offset_map = metadata.calibration_constant.data\n",
- "\n",
- "## noise\n",
- "\n",
- "metadata = ConstantMetaData()\n",
- "noise = Constants.jungfrau.Noise()\n",
- "\n",
- "metadata.calibration_constant = noise\n",
- "\n",
- "# set the operating condition\n",
- "condition = Conditions.Dark.jungfrau(memory_cells=1, bias_voltage=bias_voltage,\n",
- " integration_time=integration_time)\n",
- "device = Detectors.jungfrau1\n",
- "\n",
- "\n",
- "metadata.detector_condition = condition\n",
- "\n",
- "# specify the a version for this constant\n",
- "metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "metadata.retrieve(cal_db_interface, when=creation_time, timeout=cal_timeout)\n",
- "noise_map = metadata.calibration_constant.data\n",
- "\n",
- "## bad pixels \n",
- "\n",
- "metadata = ConstantMetaData()\n",
- "bpix = Constants.jungfrau.BadPixelsDark()\n",
- "metadata.calibration_constant = bpix\n",
- "\n",
- "# set the operating condition\n",
- "condition = Conditions.Dark.jungfrau(memory_cells=1, bias_voltage=bias_voltage,\n",
- " integration_time=integration_time)\n",
- "device = Detectors.jungfrau1\n",
- "\n",
- "\n",
- "metadata.detector_condition = condition\n",
- "\n",
- "# specify the a version for this constant\n",
- "metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "metadata.retrieve(cal_db_interface, when=creation_time, timeout=cal_timeout)\n",
- "mask = metadata.calibration_constant.data"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "def copy_and_sanitize_non_cal_data(infile, outfile, h5base):\n",
- " \"\"\" Copy and sanitize data in `infile` that is not touched by `correctLPD`\n",
- " \"\"\"\n",
- " \n",
- " if h5base.startswith(\"/\"):\n",
- " h5base = h5base[1:]\n",
- " dont_copy = [\"adc\",]\n",
- " dont_copy = [h5base+\"/{}\".format(do)\n",
- " for do in dont_copy]\n",
- "\n",
- " def visitor(k, item):\n",
- " if k not in dont_copy:\n",
- " if isinstance(item, h5py.Group):\n",
- " outfile.create_group(k)\n",
- " elif isinstance(item, h5py.Dataset):\n",
- " group = str(k).split(\"/\")\n",
- " group = \"/\".join(group[:-1])\n",
- " infile.copy(k, outfile[group])\n",
- "\n",
- " infile.visititems(visitor)\n",
- " "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import copy\n",
- "offset_map = np.squeeze(offset_map)\n",
- "mask = np.squeeze(mask)\n",
- "\n",
- "fim_data = None\n",
- "gim_data = None\n",
- "rim_data = None\n",
- "msk_data = None\n",
- "for k, f in enumerate(file_list):\n",
- " with h5py.File(f, 'r') as infile:\n",
- " out_file = \"{}/{}\".format(out_folder, f.split(\"/\")[-1])\n",
- " out_file = out_file.replace(\"RAW\", \"CORR\")\n",
- " with h5py.File(out_file, \"w\") as ofile:\n",
- " copy_and_sanitize_non_cal_data(infile, ofile, h5path)\n",
- " d = infile[h5path+\"/adc\"][()].astype(np.float32)\n",
- " if k == 0:\n",
- " rim_data = np.squeeze(copy.copy(d))\n",
- " g = infile[h5path+\"/gain\"][()]\n",
- " oshape = d.shape\n",
- " ddset = ofile.create_dataset(h5path+\"/adc\",\n",
- " oshape,\n",
- " chunks=(chunk_size_idim, 1, oshape[1], oshape[2]),\n",
- " dtype=np.float32)\n",
- " \n",
- " mskset = ofile.create_dataset(h5path+\"/mask\",\n",
- " oshape,\n",
- " chunks=(chunk_size_idim, 1, oshape[1], oshape[2]),\n",
- " dtype=np.uint32,\n",
- " compression=\"gzip\", compression_opts=1, shuffle=True)\n",
- " g[g==3] = 2\n",
- " \n",
- " offset = np.choose(g, (offset_map[...,0], offset_map[...,1], offset_map[...,2]))\n",
- " d -= offset\n",
- " ddset[...] = d\n",
- " \n",
- " msk = np.choose(g, (mask[...,0], mask[...,1], mask[...,2]))\n",
- " mskset[...] = msk\n",
- " if k == 0:\n",
- " fim_data = np.squeeze(copy.copy(d))\n",
- " gim_data = np.squeeze(copy.copy(g))\n",
- " msk_data = np.squeeze(copy.copy(msk))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "def do_2d_plot(data, edges, y_axis, x_axis):\n",
- " from matplotlib.colors import LogNorm\n",
- " fig = plt.figure(figsize=(10,10))\n",
- " ax = fig.add_subplot(111)\n",
- " extent = [np.min(edges[1]), np.max(edges[1]),np.min(edges[0]), np.max(edges[0])]\n",
- " im = ax.imshow(data[::-1,:], extent=extent, aspect=\"auto\", norm=LogNorm(vmin=1, vmax=np.max(data)))\n",
- " ax.set_xlabel(x_axis)\n",
- " ax.set_ylabel(y_axis)\n",
- " cb = fig.colorbar(im)\n",
- " cb.set_label(\"Counts\")\n",
- " "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "h, ex, ey = np.histogram2d(rim_data.flatten(), gim_data.flatten(),\n",
- " bins=[100, 4], range=[[0, 10000], [0,4]])\n",
- "do_2d_plot(h, (ex, ey), \"Signal (ADU)\", \"Gain Bit Value\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Mean RAW Preview ###\n",
- "\n",
- "The per pixel mean of the sequence file of RAW data"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "fig = plt.figure(figsize=(20,10))\n",
- "ax = fig.add_subplot(111)\n",
- "im = ax.imshow(np.mean(rim_data,axis=0),\n",
- " vmin=min(0.75*np.median(rim_data[rim_data > 0]), 2000),\n",
- " vmax=max(1.5*np.median(rim_data[rim_data > 0]), 8000), cmap=\"jet\")\n",
- "cb = fig.colorbar(im, ax=ax)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Mean CORRECTED Preview ###\n",
- "\n",
- "The per pixel mean of the sequence file of CORR data"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "fig = plt.figure(figsize=(20,10))\n",
- "ax = fig.add_subplot(111)\n",
- "im = ax.imshow(np.mean(fim_data,axis=0),\n",
- " vmin=min(0.75*np.median(fim_data[fim_data > 0]), -100),\n",
- " vmax=max(1.5*np.median(fim_data[fim_data > 0]), 100), cmap=\"jet\")\n",
- "cb = fig.colorbar(im, ax=ax)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Single Train Preview ###\n",
- "\n",
- "A single image from the first train"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "fig = plt.figure(figsize=(20,10))\n",
- "ax = fig.add_subplot(111)\n",
- "im = ax.imshow(fim_data[0,...],\n",
- " vmin=min(0.75*np.median(fim_data[0,...]), -100),\n",
- " vmax=max(1.5*np.median(fim_data[0,...]), 100), cmap=\"jet\")\n",
- "cb = fig.colorbar(im, ax=ax)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Signal Distribution ##"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "fig = plt.figure(figsize=(20,10))\n",
- "ax = fig.add_subplot(211)\n",
- "h = ax.hist(fim_data.flatten(), bins=1000, range=(-100, 1000), log=True)\n",
- "l = ax.set_xlabel(\"Signal (ADU)\")\n",
- "l = ax.set_ylabel(\"Counts\")\n",
- "\n",
- "ax = fig.add_subplot(212)\n",
- "h = ax.hist(fim_data.flatten(), bins=1000, range=(-100, 10000), log=True)\n",
- "l = ax.set_xlabel(\"Signal (ADU)\")\n",
- "l = ax.set_ylabel(\"Counts\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Maximum GAIN Preview ###\n",
- "\n",
- "The per pixel maximum of the first train of the GAIN data"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "fig = plt.figure(figsize=(20,10))\n",
- "ax = fig.add_subplot(111)\n",
- "im = ax.imshow(np.max(gim_data, axis=0), vmin=0,\n",
- " vmax=3, cmap=\"jet\")\n",
- "cb = fig.colorbar(im, ax=ax)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Bad Pixels ##\n",
- "The mask contains dedicated entries for all pixels and memory cells as well as all three gains stages. Each mask entry is encoded in 32 bits as:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "from cal_tools.enums import BadPixels\n",
- "from IPython.display import HTML, display, Markdown, Latex\n",
- "import tabulate\n",
- "table = []\n",
- "for item in BadPixels:\n",
- " table.append((item.name, \"{:016b}\".format(item.value)))\n",
- "md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=[\"Bad pixel type\", \"Bit mask\"])))"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Single Image Bad Pixels ###\n",
- "\n",
- "A single image bad pixel map for the first image of the first train"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "fig = plt.figure(figsize=(20,10))\n",
- "ax = fig.add_subplot(111)\n",
- "im = ax.imshow(np.log2(msk_data[0,...]), vmin=0, vmax=32, cmap=\"jet\")\n",
- "cb = fig.colorbar(im, ax=ax)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": []
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.6"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index 309153da8010d533378708f383a0881793bfdd6c..8d4b605c01a3d87cbc9439918c53eb630b5cc26a 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -15,33 +15,38 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true
+ "collapsed": false
},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/FXE/201801/p002112/raw/\" # the folder to read data from, required\n",
- "run = 16 # runs to process, required\n",
- "# out_folder = \"/gpfs/exfel/data/scratch/haufs/test/\" # the folder to output to, required\n",
- "out_folder = \"/gpfs/exfel/exp/HSLAB/201830/p900017/proc/test/\"\n",
+ "in_folder = \"/gpfs/exfel/exp/FXE/201802/p002271/ra\" # the folder to read data from, required\n",
+ "run = 132 # runs to process, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/xcal/test/\" # the folder to output to, required\n",
"calfile = \"\" # path to calibration file. Leave empty if all data should come from DB, not actually used, makes automode happy\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"mem_cells = 1 # memory cells in data, not actually used, makes automode happy\n",
"overwrite = True # set to True if existing data should be overwritten\n",
- "no_relative_gain = True # do not do relative gain correction\n",
- "cluster_profile = \"noDB\"\n",
+ "no_relative_gain = False # do not do relative gain correction\n",
+ "cluster_profile = \"noDB\" # cluster profile to use\n",
"bias_voltage = 90 # will be overwritten by value in file\n",
- "cal_db_interface = \"tcp://max-exfl016:8018\" #\"tcp://max-exfl016:8015#8025\" # the database interface to use\n",
- "use_dir_creation_date = False # use the creation data of the input dir for database queries\n",
- "sequences_per_node = 10 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel\n",
+ "cal_db_interface = \"tcp://max-exfl016:8016\" #\"tcp://max-exfl016:8015#8025\" # the database interface to use\n",
+ "use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
+ "sequences_per_node = 5 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel\n",
"photon_energy = 9.2 # photon energy in keV\n",
"nodb = False # if set only file-based constants will be used\n",
"chunk_size_idim = 1 # chunking size of imaging dimension, adjust if user software is sensitive to this.\n",
- "path_template = 'RAW-R{:04d}-DA06-S000{{:02d}}.h5' # template to use for file name, double escape sequence number\n",
+ "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # template to use for file name, double escape sequence number\n",
+ "path_template_control = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # template to use for file name, double escape sequence number\n",
"integration_time = 1000 # integration time in us, will be overwritten by value in file\n",
- "h5path = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data' # path in H5 file under which images are located\n",
- "gmapfile = \"/gpfs/exfel/exp/HSLAB/201831/p900053/proc/M039/gainMaps_M039.h5\" #temporary gain calibration file, not in the DB; leave empty if using DB\n",
- "memcells = 1 \n",
- "karabo_id = \"FXE_XAD_JF500K\"\n",
+ "h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data' # path in H5 file under which images are located\n",
+ "gmapfile = \"/gpfs/exfel/data/scratch/xcal/jfgain/gainMaps_M233.h5\" #temporary gain calibration file, not in the DB; leave empty if using DB\n",
+ "memcells = 1 # number of memory cells\n",
+ "karabo_id = \"FXE_XAD_JF1M\" # karabo prefix of Jungfrau devices\n",
+ "receiver_id = \"RECEIVER\" # inset for receiver devices\n",
+ "control_id = \"CONTROL\" # inset for control devices\n",
+ "db_module = \"Jungfrau_M233\" # ID of module in calibration database\n",
+ "path_inset = \"JNGFR02\" # file inset for image data\n",
+ "path_inset_control = \"JNGFR01\" # file inset for control data\n",
"\n",
"def balance_sequences(in_folder, run, sequences, sequences_per_node):\n",
" import glob\n",
@@ -65,14 +70,23 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
},
"outputs": [],
"source": [
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
- "h5path = h5path.format(karabo_id)\n",
+ "h5path = h5path.format(karabo_id, receiver_id)\n",
"import matplotlib\n",
"matplotlib.use('agg')\n",
"import matplotlib.pyplot as plt\n",
@@ -119,9 +133,12 @@
"\n",
"\n",
"\n",
- "fp_name = path_template.format(run)\n",
+ "fp_name = path_template.format(run, path_inset)\n",
"fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
"\n",
+ "fp_name_contr = path_template_control.format(run, path_inset_control)\n",
+ "fp_path_contr = '{}/{}'.format(ped_dir, fp_name_contr)\n",
+ "\n",
"\n",
"\n",
"filep_size = 500\n",
@@ -152,9 +169,9 @@
"outputs": [],
"source": [
"import h5py\n",
- "with h5py.File(fp_path.format(0), 'r') as f:\n",
- " integration_time = int(f['/RUN/{}/DET/CONTROL/exposureTime/value'.format(karabo_id)][()]*1e6)\n",
- " bias_voltage = int(np.squeeze(f['/RUN/{}/DET/CONTROL/vHighVoltage/value'.format(karabo_id)])[0])\n",
+ "with h5py.File(fp_path_contr.format(0), 'r') as f:\n",
+ " integration_time = int(f['/RUN/{}/DET/{}/exposureTime/value'.format(karabo_id, control_id)][()]*1e6)\n",
+ " bias_voltage = int(np.squeeze(f['/RUN/{}/DET/{}/vHighVoltage/value'.format(karabo_id, control_id)])[0])\n",
" print(\"Integration time is {} us\".format(integration_time))\n",
" print(\"Bias voltage is {} V\".format(bias_voltage))"
]
@@ -179,13 +196,13 @@
" if sequences is None:\n",
" for seq in range(len(dirlist)):\n",
" \n",
- " if path_template.format(run).format(seq) in abs_entry:\n",
+ " if path_template.format(run, path_inset).format(seq) in abs_entry:\n",
" file_list.append(abs_entry)\n",
" total_sequences += 1\n",
" fsequences.append(seq)\n",
" else:\n",
" for seq in sequences:\n",
- " if path_template.format(run).format(seq) in abs_entry:\n",
+ " if path_template.format(run, path_inset).format(seq) in abs_entry:\n",
" file_list.append(os.path.abspath(abs_entry))\n",
" total_sequences += 1\n",
" fsequences.append(seq)\n",
@@ -248,14 +265,17 @@
"# set the operating condition\n",
"condition = Conditions.Dark.jungfrau(memory_cells=1, bias_voltage=bias_voltage,\n",
" integration_time=integration_time)\n",
- "device = Detectors.jungfrau1\n",
- "\n",
- "\n",
+ "device = getattr(Detectors, db_module)\n",
"metadata.detector_condition = condition\n",
"\n",
"# specify the a version for this constant\n",
- "metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "metadata.retrieve(cal_db_interface, when=creation_time, timeout=cal_timeout)\n",
+ "if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " metadata.retrieve(cal_db_interface)\n",
+ "else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
+ " start=creation_time)\n",
+ " metadata.retrieve(cal_db_interface, when=creation_time.isoformat(), timeout=3000000)\n",
"offset_map = metadata.calibration_constant.data\n",
"\n",
"## noise\n",
@@ -268,14 +288,19 @@
"# set the operating condition\n",
"condition = Conditions.Dark.jungfrau(memory_cells=1, bias_voltage=bias_voltage,\n",
" integration_time=integration_time)\n",
- "device = Detectors.jungfrau1\n",
+ "\n",
"\n",
"\n",
"metadata.detector_condition = condition\n",
"\n",
"# specify the a version for this constant\n",
- "metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "metadata.retrieve(cal_db_interface, when=creation_time, timeout=cal_timeout)\n",
+ "if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " metadata.retrieve(cal_db_interface)\n",
+ "else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
+ " start=creation_time)\n",
+ " metadata.retrieve(cal_db_interface, when=creation_time.isoformat(), timeout=3000000)\n",
"noise_map = metadata.calibration_constant.data\n",
"\n",
"## bad pixels \n",
@@ -287,14 +312,19 @@
"# set the operating condition\n",
"condition = Conditions.Dark.jungfrau(memory_cells=1, bias_voltage=bias_voltage,\n",
" integration_time=integration_time)\n",
- "device = Detectors.jungfrau1\n",
+ "\n",
"\n",
"\n",
"metadata.detector_condition = condition\n",
"\n",
"# specify the a version for this constant\n",
- "metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "metadata.retrieve(cal_db_interface, when=creation_time, timeout=cal_timeout)\n",
+ "if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " metadata.retrieve(cal_db_interface)\n",
+ "else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
+ " start=creation_time)\n",
+ " metadata.retrieve(cal_db_interface, when=creation_time.isoformat(), timeout=3000000)\n",
"mask = metadata.calibration_constant.data\n",
"\n",
"\n"
@@ -304,7 +334,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": true
},
"outputs": [],
"source": [
diff --git a/notebooks/Jungfrau/Jungfrau_dark_analysis_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_dark_analysis_NBC.ipynb
deleted file mode 100644
index 03f2e32eff718eb2b6ff56b0bfd2fddeb1c97cfb..0000000000000000000000000000000000000000
--- a/notebooks/Jungfrau/Jungfrau_dark_analysis_NBC.ipynb
+++ /dev/null
@@ -1,442 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Jungfrau Dark Image Characterization #\n",
- "\n",
- "Version: 0.1, Author: M. Ramilli, S. Hauf\n",
- "\n",
- "Analyzes Jungfrau dark image data to deduce offset, noise and resulting bad pixel maps"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "in_folder = '/gpfs/exfel/exp/FXE/201801/p002118/raw/' # folder under which runs are located, required\n",
- "out_folder = '' # path to place reports at, required\n",
- "dark_run = 4 # the run number the data is located in, required\n",
- "sequences = 1 # number of sequence files in that run\n",
- "path_template = 'RAW-R{:04d}-DA06-S000{{:02d}}.h5' # template to use for file name, double escape sequence number\n",
- "cluster_profile = 'noDB' # the ipcluster profile name\n",
- "cal_db_interface = 'tcp://max-exfl016:8016' # calibrate db interface to connect to\n",
- "integration_time = 1000 # integration time in us, will be overwritten by value in file\n",
- "bias_voltage = 90 # sensor bias voltage in V, will be overwritten by value in file\n",
- "badpixel_threshold_sigma = 20. # bad pixels defined by values outside n times this std from median\n",
- "offset_abs_threshold = [1000, 8000] # absolute bad pixel threshold in terms of offset\n",
- "chunkSize = 10 # iteration chunk size, needs to match or be less than number of images in a sequence file\n",
- "imageRange = [0, 500] # image range in which to evaluate\n",
- "memoryCells = 1 # number of memory cells\n",
- "h5path = '/INSTRUMENT/FXE_XAD_JF1M1/DET/RECEIVER:daqOutput/data' # path in H5 file under which images are located"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import warnings\n",
- "warnings.filterwarnings('ignore')\n",
- "\n",
- "\n",
- "import matplotlib\n",
- "matplotlib.use('agg')\n",
- "import numpy as np\n",
- "import XFELDetAna.xfelpyanatools as xana\n",
- "import XFELDetAna.xfelpycaltools as xcal\n",
- "\n",
- "from cal_tools.enums import BadPixels\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "\n",
- "\n",
- "\n",
- "from XFELDetAna.util import env\n",
- "env.iprofile = cluster_profile\n",
- "\n",
- "from XFELDetAna.detectors.jungfrau import readerPSI as jfreaderPSI\n",
- "from XFELDetAna.detectors.jungfrau import reader as jfreader\n",
- "from XFELDetAna.detectors.jungfrau.jf_chunk_reader import JFChunkReader\n",
- "from XFELDetAna.detectors.jungfrau.util import non_empty_trains\n",
- "\n",
- "# so constants relevant for the analysis\n",
- "run = dark_run\n",
- "cpuCores = 1\n",
- "is_parallel = False\n",
- "sensorSize = [1024, 512]\n",
- "blockSize = [1024, 512]\n",
- "xRange = [0, 0+sensorSize[0]]\n",
- "yRange = [0, 0+sensorSize[1]]\n",
- "gains = [0, 1, 2]\n",
- "\n",
- "ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "fp_name = path_template.format(run)\n",
- "fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
- "\n",
- "\n",
- "\n",
- "filep_size = 500\n",
- "myRange_P = range(0, sequences)\n",
- "path = h5path\n",
- "\n",
- "print(\"Reading data from {}\".format(fp_path))\n",
- "print(\"Run is: {}\".format(run))\n",
- "print(\"HDF5 path: {}\".format(h5path))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import h5py\n",
- "with h5py.File(fp_path.format(0), 'r') as f:\n",
- " integration_time = int(f['/RUN/FXE_XAD_JF1M1/DET/CONTROL/exposureTime/value'][()]*1e6)\n",
- " bias_voltage = int(np.squeeze(f['/RUN/FXE_XAD_JF1M1/DET/CONTROL/vHighVoltage/value'])[0])\n",
- " print(\"Integration time is {} us\".format(integration_time))\n",
- " print(\"Bias voltage is {} V\".format(bias_voltage))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "reader = JFChunkReader(filename = fp_path, readFun = jfreader.readData, size = filep_size, chunkSize = chunkSize,\n",
- " path = path, image_range=imageRange, pixels_x = sensorSize[0], pixels_y = sensorSize[1],\n",
- " x_range = xRange, y_range = yRange, imagesPerChunk=chunkSize, filesRange = myRange_P,\n",
- " memoryCells=memoryCells, blockSize=blockSize)\n",
- "\n",
- "noiseCal = xcal.NoiseCalculator(sensorSize, nCells=memoryCells, cores=cpuCores, parallel=is_parallel, gains=gains,\n",
- " blockSize=blockSize)\n",
- "\n",
- "for data in reader.readChunks():\n",
- " \n",
- " images = np.squeeze(data[0])\n",
- " gainmaps = np.squeeze(data[1])\n",
- " trainId = np.array(data[2])\n",
- " idxs = non_empty_trains(trainId) \n",
- " noiseCal.fill(data=images[..., idxs], gainMap=gainmaps[..., idxs])\n",
- "\n",
- "offset_map = noiseCal.getOffset()\n",
- "noise_map = noiseCal.get()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Offset and Noise Maps ##\n",
- "\n",
- "Below offset and noise maps for the high ($g_0$) gain stage are shown, alongside the distribution of these values. One expects block-like structures mapping to the ASICs of the detector"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false,
- "scrolled": false
- },
- "outputs": [],
- "source": [
- "import matplotlib.pyplot as plt\n",
- "%matplotlib inline\n",
- "\n",
- "from XFELDetAna.plotting.histogram import histPlot\n",
- "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
- "g_idx = 0\n",
- "\n",
- "g_name = ['G0', 'G1', 'G2']\n",
- "g_range = [(0, 8000), (8000, 16000), (8000, 16000)]\n",
- "\n",
- "off = offset_map[:, :, 0, g_idx]\n",
- "fo_0 = heatmapPlot(np.swapaxes(off, 0, 1), \n",
- " y_label=\"Row\",\n",
- " x_label=\"Column\",\n",
- " lut_label=\"Pedestal {} [ADCu]\".format(g_name[g_idx]),\n",
- " vmin=g_range[g_idx][0],\n",
- " vmax=g_range[g_idx][1])\n",
- "\n",
- "fo_01, axo_01 = plt.subplots()\n",
- "ho0, eo0 , xo0, stato0 = histPlot(axo_01, off,\n",
- " bins=800,\n",
- " range=g_range[g_idx],\n",
- " facecolor='b',\n",
- " histotype='stepfilled')\n",
- "\n",
- "axo_01.tick_params(axis='both',which='major',labelsize=15)\n",
- "axo_01.set_title('Module pedestal distribution', fontsize=15)\n",
- "axo_01.set_xlabel('Pedestal {} [ADCu]'.format(g_name[g_idx]),fontsize=15)\n",
- "axo_01.set_yscale('log')\n",
- "\n",
- "noise = noise_map[:, :, 0, g_idx]\n",
- "fn_0 = heatmapPlot(np.swapaxes(noise, 0, 1), \n",
- " y_label=\"Row\",\n",
- " x_label=\"Column\",\n",
- " lut_label=\"RMS noise {} [ADCu]\".format(g_name[g_idx]),\n",
- " vmin=0,\n",
- " vmax=50)\n",
- "\n",
- "fn_01, axn_01 = plt.subplots()\n",
- "hn0, en0 , xn0, statn0 = histPlot(axn_01, noise,\n",
- " bins=100,\n",
- " range=(0, 50),\n",
- " facecolor='b',\n",
- " histotype='stepfilled')\n",
- "\n",
- "axn_01.tick_params(axis='both',which='major',labelsize=15)\n",
- "axn_01.set_title('Module noise distribution', fontsize=15)\n",
- "axn_01.set_xlabel('RMS noise {} [ADCu]'.format(g_name[g_idx]),fontsize=15)\n",
- "axn_01.set_yscale('log')\n",
- "\n",
- "plt.show()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Bad Pixel Map ###\n",
- "\n",
- "The bad pixel map is deduced by comparing offset and noise of each pixel ($v_i$) and each gain ($g$) against the median value for that gain stage:\n",
- "\n",
- "$$ \n",
- "v_i > \\mathrm{median}(v_{k,g}) + n \\sigma_{v_{k,g}}\n",
- "$$\n",
- "or\n",
- "$$\n",
- "v_i < \\mathrm{median}(v_{k,g}) - n \\sigma_{v_{k,g}} \n",
- "$$\n",
- "\n",
- "Values are encoded in a 32 bit mask, where for the dark image deduced bad pixels the following non-zero entries are relevant:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "def print_bp_entry(bp):\n",
- " print(\"{:<30s} {:032b}\".format(bp.name, bp.value))\n",
- "\n",
- "print_bp_entry(BadPixels.OFFSET_OUT_OF_THRESHOLD)\n",
- "print_bp_entry(BadPixels.NOISE_OUT_OF_THRESHOLD)\n",
- "print_bp_entry(BadPixels.OFFSET_NOISE_EVAL_ERROR)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "bad_pixels_map = np.zeros(noise_map.shape, np.uint32)\n",
- "def eval_bpidx(d):\n",
- " \n",
- " mdn = np.nanmedian(d, axis=(0,1,2))[None,None,None,:]\n",
- " std = np.nanstd(d, axis=(0,1,2))[None,None,None,:]\n",
- " \n",
- " idx = (d > badpixel_threshold_sigma*std+mdn) | (d < (-badpixel_threshold_sigma)*std+mdn)\n",
- " return idx\n",
- " \n",
- "bad_pixels_map[eval_bpidx(offset_map)] = BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
- "bad_pixels_map[~np.isfinite(offset_map)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
- "bad_pixels_map[eval_bpidx(noise_map)] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
- "bad_pixels_map[~np.isfinite(noise_map)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
- "bad_pixels_map[(offset_map < offset_abs_threshold[0]) | (offset_map > offset_abs_threshold[1])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
- "\n",
- "bad_pixels = bad_pixels_map[:, :, 0, g_idx]\n",
- "\n",
- "fn_0 = heatmapPlot(np.swapaxes(bad_pixels, 0, 1), \n",
- " y_label=\"Row\",\n",
- " x_label=\"Column\",\n",
- " lut_label=\"Badpixels {} [ADCu]\".format(g_name[g_idx]),\n",
- " vmin=0)\n",
- "\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "## offset\n",
- "\n",
- "metadata = ConstantMetaData()\n",
- "offset = Constants.jungfrau.Offset()\n",
- "offset.data = np.moveaxis(offset_map, 0, 1)\n",
- "metadata.calibration_constant = offset\n",
- "\n",
- "# set the operating condition\n",
- "condition = Conditions.Dark.jungfrau(memory_cells=1, bias_voltage=bias_voltage,\n",
- " integration_time=integration_time)\n",
- "device = Detectors.jungfrau1\n",
- "\n",
- "\n",
- "metadata.detector_condition = condition\n",
- "\n",
- "# specify the a version for this constant\n",
- "metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "metadata.send(cal_db_interface)\n",
- "\n",
- "## noise\n",
- "\n",
- "metadata = ConstantMetaData()\n",
- "noise = Constants.jungfrau.Noise()\n",
- "noise.data = np.moveaxis(noise_map, 0, 1)\n",
- "metadata.calibration_constant = noise\n",
- "\n",
- "# set the operating condition\n",
- "condition = Conditions.Dark.jungfrau(memory_cells=1, bias_voltage=bias_voltage,\n",
- " integration_time=integration_time)\n",
- "device = Detectors.jungfrau1\n",
- "\n",
- "\n",
- "metadata.detector_condition = condition\n",
- "\n",
- "# specify the a version for this constant\n",
- "metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "metadata.send(cal_db_interface)\n",
- "\n",
- "\n",
- "## bad pixels \n",
- "\n",
- "metadata = ConstantMetaData()\n",
- "bpix = Constants.jungfrau.BadPixelsDark()\n",
- "bpix.data = np.moveaxis(bad_pixels_map, 0, 1)\n",
- "metadata.calibration_constant = bpix\n",
- "\n",
- "# set the operating condition\n",
- "condition = Conditions.Dark.jungfrau(memory_cells=1, bias_voltage=bias_voltage,\n",
- " integration_time=integration_time)\n",
- "device = Detectors.jungfrau1\n",
- "\n",
- "\n",
- "metadata.detector_condition = condition\n",
- "\n",
- "# specify the a version for this constant\n",
- "metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "metadata.send(cal_db_interface)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Verification ##\n",
- "\n",
- "For verification we correct the input data with the offset deduced from it. We expect a distribution centered around zero."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "\n",
- "fd_path = fp_path\n",
- "d_path = path\n",
- "\n",
- "filed_size = 500\n",
- "myRange_D = range(0, 2)\n",
- "imageRange = [0, 500]\n",
- "chunkSize = 25\n",
- "\n",
- "\n",
- "reader = JFChunkReader(filename = fd_path, readFun = jfreader.readData, size = filed_size, chunkSize = chunkSize,\n",
- " path = d_path, image_range=imageRange, pixels_x = sensorSize[0], pixels_y = sensorSize[1],\n",
- " x_range = xRange, y_range = yRange, imagesPerChunk=chunkSize, filesRange = myRange_D,\n",
- " memoryCells=memoryCells, blockSize=blockSize)\n",
- "\n",
- "offsetCorr = xcal.OffsetCorrection(shape=sensorSize, offsetMap=offset_map, nCells=memoryCells,\n",
- " cores=cpuCores, parallel=is_parallel, gains=gains, blockSize=blockSize)\n",
- "\n",
- "pixel_data = []\n",
- "\n",
- "for data_raw in reader.readChunks():\n",
- " images = np.array(data_raw[0]).astype(np.float32)\n",
- " gainmaps = np.array(data_raw[1])\n",
- " trainID = np.array(data_raw[2])\n",
- " idxs = non_empty_trains(trainId)\n",
- " data_out = offsetCorr.correct(data=images[..., idxs], gainMap=gainmaps[..., idxs])\n",
- " \n",
- " pixel_data.append(data_out)\n",
- "\n",
- "pixel_data = np.array(pixel_data)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "is_log = True\n",
- "h_range = (-500, 500)\n",
- "h_bins = 300\n",
- "\n",
- "f_sp, ax_sp = plt.subplots()\n",
- "h, e , x, stat = histPlot(ax_sp, pixel_data.flatten(),\n",
- " bins=h_bins,\n",
- " range=h_range,\n",
- " facecolor='b',\n",
- " log=is_log,\n",
- " histotype='stepfilled')\n",
- "\n",
- "ax_sp.tick_params(axis='both',which='major',labelsize=15)\n",
- "ax_sp.set_xlabel('Energy [ADCu]', fontsize=15)\n",
- "\n",
- "plt.show()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.6"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}
diff --git a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_NBC.ipynb
index 352b37088ce0b5fb41245710c80ee99e9a472827..988f254de445d561d7def3b4c13dbd61642110e4 100644
--- a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_NBC.ipynb
@@ -19,24 +19,31 @@
},
"outputs": [],
"source": [
- "in_folder = '/gpfs/exfel/exp/FXE/201801/p002112/raw/' # folder under which runs are located, required\n",
+ "in_folder = '/gpfs/exfel/exp/SPB/201921/p002429/raw/' # folder under which runs are located, required\n",
"out_folder = '' # path to place reports at, required\n",
"sequences = 1 # number of sequence files in that run\n",
- "path_template = 'RAW-R{:04d}-DA06-S000{{:02d}}.h5' # template to use for file name, double escape sequence number\n",
+ "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # template to use for file name, double escape sequence number\n",
+ "path_inset = \"DA06\" # file inset for image data\n",
+ "path_inset_control = \"DA06\" # file inset for control data\n",
"cluster_profile = 'noDB' # the ipcluster profile name\n",
"cal_db_interface = 'tcp://max-exfl016:8016' # calibrate db interface to connect to\n",
"integration_time = 1000 # integration time in us, will be overwritten by value in file\n",
"bias_voltage = 90 # sensor bias voltage in V, will be overwritten by value in file\n",
"badpixel_threshold_sigma = 20. # bad pixels defined by values outside n times this std from median\n",
- "offset_abs_threshold = [[1000, 10000, 10000], [8000, 15000, 15000]] # absolute bad pixel threshold in terms of offset\n",
+ "offset_abs_threshold_low = [1000, 10000, 10000] # absolute bad pixel threshold in terms of offset, lower values\n",
+ "offset_abs_threshold_high = [8000, 15000, 15000] # absolute bad pixel threshold in terms of offset, upper values\n",
"chunkSize = 10 # iteration chunk size, needs to match or be less than number of images in a sequence file\n",
"imageRange = [0, 500] # image range in which to evaluate\n",
"memoryCells = 1 # number of memory cells\n",
- "h5path = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data' # path in H5 file under which images are located\n",
+ "h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data' # path in H5 file under which images are located\n",
"run_high = 0 # run number for G0 dark run, required\n",
"run_med = 0 # run number for G1 dark run, required\n",
"run_low = 0 # run number for G2 dark run, required\n",
- "karabo_id = \"FXE_XAD_JF500K\""
+ "karabo_id = \"FXE_XAD_JF500K\" # karabo prefix of Jungfrau devices\n",
+ "receiver_id = \"RECEIVER\" # inset for receiver devices\n",
+ "control_id = \"CONTROL\" # inset for control devices\n",
+ "db_module = \"Jungfrau_M233\" # ID of module in calibration database\n",
+ "use_dir_creation_date = True # use dir creation date"
]
},
{
@@ -57,6 +64,7 @@
"import XFELDetAna.xfelpycaltools as xcal\n",
"\n",
"from cal_tools.enums import BadPixels\n",
+ "from cal_tools.tools import get_dir_creation_date\n",
"from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
"\n",
"from XFELDetAna.util import env\n",
@@ -76,7 +84,14 @@
"xRange = [0, 0+sensorSize[0]]\n",
"yRange = [0, 0+sensorSize[1]]\n",
"gains = [0, 1, 2]\n",
- "h5path = h5path.format(karabo_id)"
+ "h5path = h5path.format(karabo_id, receiver_id)\n",
+ "\n",
+ "creation_time = None\n",
+ "if use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run_high)\n",
+ " print(\"Using {} as creation time\".format(creation_time))\n",
+ " \n",
+ "offset_abs_threshold = [offset_abs_threshold_low, offset_abs_threshold_high]"
]
},
{
@@ -105,15 +120,17 @@
" \n",
" if run is not None:\n",
" ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- " fp_name = path_template.format(run)\n",
+ " fp_name = path_template.format(run, path_inset_control)\n",
" fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
"\n",
" with h5py.File(fp_path.format(0), 'r') as f:\n",
- " integration_time = int(f['/RUN/{}/DET/CONTROL/exposureTime/value'.format(karabo_id)][()]*1e6)\n",
- " bias_voltage = int(np.squeeze(f['/RUN/{}/DET/CONTROL/vHighVoltage/value'.format(karabo_id)])[0])\n",
+ " integration_time = int(f['/RUN/{}/DET/{}/exposureTime/value'.format(karabo_id, control_id)][()]*1e6)\n",
+ " bias_voltage = int(np.squeeze(f['/RUN/{}/DET/{}/vHighVoltage/value'.format(karabo_id, control_id)])[0])\n",
" print(\"Integration time is {} us\".format(integration_time))\n",
" print(\"Bias voltage is {} V\".format(bias_voltage))\n",
"\n",
+ " fp_name = path_template.format(run, path_inset)\n",
+ " fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
" filep_size = 500\n",
" myRange_P = range(0, sequences)\n",
" path = h5path\n",
@@ -306,13 +323,18 @@
"# set the operating condition\n",
"condition = Conditions.Dark.jungfrau(memory_cells=1, bias_voltage=bias_voltage,\n",
" integration_time=integration_time)\n",
- "device = Detectors.jungfrau1\n",
+ "device = getattr(Detectors, db_module)\n",
"\n",
"\n",
"metadata.detector_condition = condition\n",
"\n",
"# specify the a version for this constant\n",
- "metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ "if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " metadata.retrieve(cal_db_interface)\n",
+ "else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
+ " start=creation_time)\n",
"metadata.send(cal_db_interface)\n",
"\n",
"## noise\n",
@@ -325,13 +347,18 @@
"# set the operating condition\n",
"condition = Conditions.Dark.jungfrau(memory_cells=1, bias_voltage=bias_voltage,\n",
" integration_time=integration_time)\n",
- "device = Detectors.jungfrau1\n",
+ "\n",
"\n",
"\n",
"metadata.detector_condition = condition\n",
"\n",
"# specify the a version for this constant\n",
- "metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ "if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " metadata.retrieve(cal_db_interface)\n",
+ "else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
+ " start=creation_time)\n",
"metadata.send(cal_db_interface)\n",
"\n",
"\n",
@@ -345,13 +372,18 @@
"# set the operating condition\n",
"condition = Conditions.Dark.jungfrau(memory_cells=1, bias_voltage=bias_voltage,\n",
" integration_time=integration_time)\n",
- "device = Detectors.jungfrau1\n",
+ "\n",
"\n",
"\n",
"metadata.detector_condition = condition\n",
"\n",
"# specify the a version for this constant\n",
- "metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ "if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " metadata.retrieve(cal_db_interface)\n",
+ "else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
+ " start=creation_time)\n",
"metadata.send(cal_db_interface)"
]
},