diff --git a/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb b/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..a774ea46a5d6ed6c7b6e93f943f92551d8a7161a
--- /dev/null
+++ b/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
@@ -0,0 +1,654 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Histogramming of AGIPD FF data\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "Offline Calibration for the AGIPD Detector"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/MID/202030/p900137/proc/\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/exp/MID/202030/p900137/scratch/karnem/r0319_0322_0342_v50\" # the folder to output to, required\n",
+ "sequences = [-1] # module to consider, set to -1 for all, range allowed\n",
+ "modules = [5] # module to consider, range allowed\n",
+ "runs = [319] # list of run numbers, required\n",
+ "cells_list = ['range(0,15)'] # list of lists or any python expressions, which can be converted to a list. E.g. 'range(0,15,5)' 'list(range(0,5))+list(range(50,60))'\n",
+ "\n",
+ "karabo_id = \"MID_DET_AGIPD1M-1\" # karabo karabo_id\n",
+ "karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
+ "receiver_id = \"{}CH0\" # inset for receiver devices\n",
+ "path_template = 'CORR-R{:04d}-AGIPD{:02d}-S{}.h5' # the template to use to access data\n",
+ "h5path = 'INSTRUMENT/{}/DET/{}:xtdf/image/' # path in the HDF5 file to images\n",
+ "\n",
+ "n_bins_adu = 1 # number of bins per ADU\n",
+ "h_range = [-50, 450] # range of the histogram in ADU\n",
+ "n_cells = 202 # total number of memory cells (used to create summary file)\n",
+ "\n",
+ "hist_std_tresh = 10 # Threshold for histogram standard deviation\n",
+ "hist_mean_tresh = 15 # Threshold for histogram mean\n",
+ "h_sums_tresh = 100 # Threshold for number of entries in histograms\n",
+ "\n",
+ "n_cores_hists = 20 # Number of processes (cores) to create histograms\n",
+ "n_cores_files = 10 # Number of processes (cores) to read files\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import glob\n",
+ "from multiprocessing import Pipe, Process, Queue, Pool\n",
+ "import os\n",
+ "import sys\n",
+ "from time import sleep, time, perf_counter\n",
+ "import warnings\n",
+ "\n",
+ "import h5py\n",
+ "import matplotlib.pyplot as plt\n",
+ "import matplotlib as mpl\n",
+ "import numpy as np\n",
+ "import sharedmem\n",
+ "import traceback\n",
+ "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
+ "\n",
+ "from cal_tools.cython import agipdalgs as calgs\n",
+ "from cal_tools.ana_tools import save_dict_to_hdf5\n",
+ "\n",
+ "%matplotlib inline\n",
+ "warnings.filterwarnings('ignore')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if karabo_da[0] != '-1':\n",
+ " module = int(karabo_da[0][-2:])\n",
+ "else:\n",
+ " module = modules[0]\n",
+ " \n",
+ "n_pix = 128*512\n",
+ "h5path = h5path.format(karabo_id, receiver_id)\n",
+ "n_bins = (h_range[1]-h_range[0])*n_bins_adu\n",
+ "print(f\"Number of bins: {n_bins}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# loop over runs\n",
+ "file_to_read = Queue()\n",
+ "processed = {}\n",
+ "n_files = 0\n",
+ "for irun, run in enumerate(runs):\n",
+ " \n",
+ " print(f\"Run {run}\")\n",
+ " file_path = f\"{in_folder}/r{run:04d}/{path_template}\"\n",
+ " print(f'File path template: {file_path}')\n",
+ "\n",
+ " cells = list(eval(cells_list[irun]))\n",
+ " if len(cells)>0:\n",
+ " print(f'List of cells {cells[0]}-{cells[-1]}: {cells}')\n",
+ " else:\n",
+ " print('List of cells is empty. Nothing to process.')\n",
+ " continue\n",
+ "\n",
+ " if sequences[0] == -1:\n",
+ " fnames = glob.glob(file_path.format(run, module, '*'))\n",
+ " else:\n",
+ " fnames = []\n",
+ " for sequence in sequences:\n",
+ " fname = file_path.format(run, module, f'{sequence:05d}')\n",
+ " if os.path.isfile(fname):\n",
+ " fnames.append(fname)\n",
+ " \n",
+ " if not fnames:\n",
+ " print('No files found.')\n",
+ " continue\n",
+ " \n",
+ " run_sequences = [int(x.split('/')[-1].split('-')[3][1:-3]) for x in fnames]\n",
+ " print(f\"List of sequences: {sorted(run_sequences)}\")\n",
+ " \n",
+ " processed[run] = sharedmem.empty((max(run_sequences+[1])+1, max(cells+[1])+1), dtype='i4')\n",
+ " n_files += len(fnames)\n",
+ " for file in fnames:\n",
+ " file_to_read.put((file, np.array(cells)))\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Allocate shared memory for files and histograms\n",
+ "total_hist = sharedmem.empty((n_cells, n_bins, n_pix), dtype='i4')\n",
+ "total_img = sharedmem.empty((n_cells), dtype='i4')\n",
+ "timing = sharedmem.empty((n_cores_files, 3, 2), dtype='f4')\n",
+ "\n",
+ "shared_data = {}\n",
+ "for i in range(n_cores_files):\n",
+ " shared_data[i] = {}\n",
+ " shared_data[i]['data'] = sharedmem.empty((256*n_cells, 512, 128), dtype='f4')\n",
+ " shared_data[i]['cellId'] = sharedmem.empty((256*n_cells), dtype='i4')\n",
+ " shared_data[i]['nImg'] = sharedmem.empty(1, dtype='i4') \n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def read_file(file_name, cells, data_path, i_proc):\n",
+ " \"\"\"\n",
+ " Read images and cell Ids from file to the shared memory\n",
+ "\n",
+ " :param file_name: (str) Name of input file including path\n",
+ " :param cells: (numpy.ndarray, rank=1) array of cells to consider\n",
+ " :param data_path: (str) Path to data inside of h5 file\n",
+ " :param i_proc: (int) Index of reader process and corresponding \n",
+ " index of shared memory array\n",
+ "\n",
+ " \"\"\"\n",
+ "\n",
+ " global shared_data\n",
+ " with h5py.File(file_name, 'r') as f:\n",
+ " group = f[data_path]\n",
+ "\n",
+ " cell_id = np.squeeze(group['cellId'][()])\n",
+ " train_id = np.squeeze(group['trainId'][()])\n",
+ " sel = np.nonzero(np.in1d(cell_id, cells) & (train_id > 0))[0]\n",
+ " n_img = len(sel)\n",
+ " shared_data[i_proc]['data'][:n_img] = group['data'][()][sel]\n",
+ " shared_data[i_proc]['cellId'][:n_img] = group['cellId'][sel]\n",
+ " shared_data[i_proc]['nImg'][:] = n_img\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def fill_hist(cells, i_proc):\n",
+ " \"\"\"\n",
+ " Calculate histograms and add it to total histograms\n",
+ " \n",
+ " :param cells: (numpy.ndarray, rank=1) array of cells to consider\n",
+ " :param i_proc: (int) Index of reader process and corresponding \n",
+ " index of shared memory array\n",
+ "\n",
+ " \"\"\"\n",
+ "\n",
+ " global shared_data\n",
+ " n_img = shared_data[i_proc]['nImg'][0]\n",
+ " for ic, cell in enumerate(cells):\n",
+ " cell_id = shared_data[i_proc]['cellId'][:n_img]\n",
+ " cell_sel = np.where((cell_id == cell))\n",
+ "\n",
+ " data = shared_data[i_proc]['data'][:n_img][cell_sel]\n",
+ " total_hist[cell] += calgs.histogram(data.reshape(data.shape[0], n_pix),\n",
+ " bins=n_bins,\n",
+ " range=h_range)[0].astype(np.int32)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Process all files\n",
+ "def file_reader_processor(conn, i_proc, file_to_read, file_to_process):\n",
+ " \"\"\"\n",
+ " Process all files in the file_to_read Queue\n",
+ " \n",
+ " Processing of file includes reading file to shared memory and \n",
+ " creating of histograms.\n",
+ " \n",
+ " :param conn: (pipe) Pipe for communication with manager\n",
+ " :param i_proc: (int) Index of reader process and corresponding \n",
+ " index of shared memory array\n",
+ " :param file_to_read: (queue) Queue of files to be loaded to shared memory\n",
+ " :param file_to_process: (queue) Queue of files in shared momery to be processed\n",
+ " with histogramm_processor\n",
+ " \n",
+ " \"\"\"\n",
+ "\n",
+ " global shared_data\n",
+ " while not file_to_read.empty():\n",
+ "\n",
+ " file_name, cells = file_to_read.get()\n",
+ " try:\n",
+ " ts = perf_counter()\n",
+ " module_idx = int(file_name.split('/')[-1].split('-')[2][-2:])\n",
+ " module_name = f\"Q{module_idx // 4 + 1}M{module_idx % 4 + 1}\"\n",
+ " run = int(file_name.split('/')[-1].split('-')[1][-4:])\n",
+ " sequence = int(file_name.split('/')[-1].split('-')[3][1:-3])\n",
+ " data_path = h5path.format(module_idx)\n",
+ " \n",
+ " read_file(file_name, cells, data_path, i_proc)\n",
+ " t_read = perf_counter()-ts\n",
+ " \n",
+ " file_to_process.put((i_proc, cells, run, sequence))\n",
+ " msg, t_hist = conn.recv()\n",
+ " t_all = perf_counter()-ts\n",
+ " \n",
+ " timing[i_proc][0,0] += t_read\n",
+ " timing[i_proc][0,1] += t_read**2\n",
+ " timing[i_proc][1,0] += t_hist\n",
+ " timing[i_proc][1,1] += t_hist**2\n",
+ " timing[i_proc][2,0] += t_all\n",
+ " timing[i_proc][2,1] += t_all**2\n",
+ " \n",
+ " except Exception as e:\n",
+ " err = f\"Error: {e}\\nError traceback: {traceback.format_exc()}\"\n",
+ " print(f'Error while processing file: \\n{file_name}')\n",
+ " print(err)\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def process_manager(conn_proc, conn_hist, file_to_process):\n",
+ " \"\"\"\n",
+ " Manager of reader and histogramming processes\n",
+ " \n",
+ " Manager discontinueally monitor a queue.\n",
+ " If queue is not empty it initiate histogramming of \n",
+ " images stored in a shared memory. When images for one file \n",
+ " are processed corresponding reader get informed to read next file.\n",
+ " \n",
+ " :param conn_proc: (list) List of pipes for communication with readers\n",
+ " :param conn_hist: (list) List of pipes for communication with histogramm_processors\n",
+ " :param file_to_process: (queue) Queue of files in shared momery to be processed\n",
+ " with histogramm_processor\n",
+ " \"\"\"\n",
+ " \n",
+ " global shared_data\n",
+ " while True:\n",
+ " if not file_to_process.empty():\n",
+ " try:\n",
+ " info = file_to_process.get()\n",
+ " if not info:\n",
+ " continue\n",
+ " i_file, cells, run, sequence = info\n",
+ " ts = perf_counter()\n",
+ " works = np.array_split(cells, n_cores_hists)\n",
+ "\n",
+ " for i, work in enumerate(works):\n",
+ " conn_hist[i].send([work, i_file])\n",
+ " \n",
+ " err = ''\n",
+ " for i in range(n_cores_hists):\n",
+ " err += conn_hist[i].recv()\n",
+ " \n",
+ " if err == '':\n",
+ " n_img = shared_data[i_file]['nImg'][0]\n",
+ " processed[run][sequence, cells] = n_img // len(cells)\n",
+ " total_img[cells] += n_img // len(cells)\n",
+ " else:\n",
+ " processed[run][sequence, cells] = 0\n",
+ "\n",
+ " except Exception as e:\n",
+ " err = f\"Error: {e}\\nError traceback: {traceback.format_exc()}\"\n",
+ " print(err)\n",
+ " \n",
+ " conn_proc[int(i_file)].send(['Done', perf_counter()-ts])\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def histogramm_processor(conn):\n",
+ " \"\"\"\n",
+ " Create histograms from images in shared memory\n",
+ " \n",
+ " Function discontinuity waiting for a message from \n",
+ " process manager. When message arrives histograms are created \n",
+ " for requested cells using images from shared memory. After \n",
+ " histograms are created message is send back to inform process manager.\n",
+ " \n",
+ " :param conn: Pipe to communicate with process_manager\n",
+ " \"\"\"\n",
+ " while True:\n",
+ " err = ''\n",
+ " try:\n",
+ " msg = conn.recv()\n",
+ " cells, i_proc = msg\n",
+ " fill_hist(cells, i_proc)\n",
+ " except Exception as e:\n",
+ " err = f\"Error: {e}\\nError traceback: {traceback.format_exc()}\"\n",
+ " print(err)\n",
+ " conn.send(err) \n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Creare start and join workers\n",
+ "file_to_process = Queue()\n",
+ "file_workers = []\n",
+ "hist_workers = []\n",
+ "manager = None\n",
+ "\n",
+ "proc_pipes = []\n",
+ "hist_pipes = []\n",
+ "for i in range(n_cores_files):\n",
+ " a, b = Pipe()\n",
+ " proc_pipes.append(b)\n",
+ " file_workers.append(Process(target=file_reader_processor, \n",
+ " args=(a, i, file_to_read, file_to_process) ) )\n",
+ " file_workers[i].start()\n",
+ " \n",
+ "for j in range(n_cores_hists):\n",
+ " a, b = Pipe()\n",
+ " hist_pipes.append(b)\n",
+ " hist_workers.append(Process(target=histogramm_processor, args=(a,) ) )\n",
+ " hist_workers[j].start()\n",
+ " \n",
+ "manager = Process(target=process_manager, args=(proc_pipes, hist_pipes, file_to_process) )\n",
+ "manager.start()\n",
+ "\n",
+ "ts = perf_counter()\n",
+ "for i in range(n_cores_files):\n",
+ " file_workers[i].join()\n",
+ "\n",
+ "manager.terminate()\n",
+ "manager.join()\n",
+ "\n",
+ "for j in range(n_cores_hists):\n",
+ " hist_workers[j].terminate()\n",
+ " hist_workers[j].join()\n",
+ "\n",
+ "sum_timing = np.sum(timing, axis=0)\n",
+ "ave_timing = sum_timing[:,0] / n_files\n",
+ "std_timing = np.sqrt((sum_timing[:,1] - sum_timing[:,0]**2/n_files)/n_files)\n",
+ "print(f'Processing of {n_files} files finished')\n",
+ "print(f\"Total processing time: {perf_counter()-ts:.01f}s\")\n",
+ "print(f'Reading time per file: {ave_timing[0]:.01f} +- {std_timing[0]:.02f}s')\n",
+ "print(f'Histogramming time per file: {ave_timing[1]:.01f} +- {std_timing[1]:.02f}s')\n",
+ "print(f'File processing time: {ave_timing[2]:.01f} +- {std_timing[2]:.02f}s')\n",
+ "del shared_data"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Processed files ###\n",
+ "\n",
+ "Plots below shows which data have been processed to create histograms."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for run in processed:\n",
+ " fig = plt.figure(figsize=(10,5))\n",
+ " ax = fig.add_subplot(111)\n",
+ "\n",
+ " _ = heatmapPlot(processed[run], add_panels=False, cmap='viridis', use_axis=ax,\n",
+ " vmin=-1, vmax=256, lut_label='Number of images')\n",
+ " ax.set_xlabel('Cells', fontsize=14)\n",
+ " ax.set_ylabel('Sequences', fontsize=14)\n",
+ " ax.set_title(f\"Processed run {run}, module {module}\", fontsize=16, fontweight='bold')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "os.makedirs(out_folder, exist_ok=True)\n",
+ "out_name = f'{out_folder}/hists_m{module:02d}_sum.h5'\n",
+ "print(f'Save to file: {out_name}')\n",
+ "save_dict_to_hdf5({'hist': total_hist,\n",
+ " 'cellId': np.arange(n_cells),\n",
+ " 'nImages': total_img,\n",
+ " 'nBins': n_bins,\n",
+ " 'hRange': np.array(h_range),\n",
+ " 'runs': processed}, out_name)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Perfomr analysis of the data quality\n",
+ "rshist = np.reshape(total_hist, (n_cells, n_bins, 512, 128))\n",
+ "\n",
+ "x = np.linspace(h_range[0], h_range[1], n_bins)\n",
+ "mids = 0.5*(x[1:] + x[:-1])\n",
+ "\n",
+ "h_sums = np.sum(rshist, axis=1)\n",
+ "hist_norm = rshist / h_sums[:, None, :, :]\n",
+ "hist_mean = np.sum(hist_norm[:, :n_bins-1, ...] *\n",
+ " mids[None, :, None, None], axis=1)\n",
+ "\n",
+ "hist_std_pix = np.nanstd(hist_mean, axis=(1, 2))\n",
+ "hist_mean_pix = np.nanmean(hist_mean, axis=(1, 2))\n",
+ "\n",
+ "h_sum_mean_pix = np.nanmean(h_sums, axis=(1, 2))\n",
+ "h_sum_std_pix = np.nanstd(h_sums, axis=(1, 2))\n",
+ "\n",
+ "hist_sqr = (mids[None, :, None, None] - hist_mean[:, None, ...])**2\n",
+ "hist_std = np.sqrt(np.sum(hist_norm[:, :n_bins-1, ...] * hist_sqr, axis=1))\n",
+ "\n",
+ "h_out = (h_sums.astype(np.float32) - total_img[:, None, None].astype(\n",
+ " np.float32))/total_img[:, None, None].astype(np.float32)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Analysis across all histograms\n",
+ "\n",
+ "Mean and standard deviation of the histograms reflect positions of photon peaks. Low value of mean may indicate significant contribution from noise peak. Low standard deviation, comparable with the value of noise may show absence of photon peaks.\n",
+ "\n",
+ "Number of entries in histograms expected to be very close to the number of processed images. In case of several processed runs, which covers different cell regions, number of images will be different for different cells. Distribution of histogram entries will have few peaks.\n",
+ "\n",
+ "Fraction of outliers expected to be nearly zero for most of the histograms. Large fraction of outliers may significantly distort estimation of position of photon peaks. This likely happen for hot pixels."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fig = plt.figure(figsize=(10, 5))\n",
+ "ax0 = fig.add_subplot(111)\n",
+ "a = ax0.hist(hist_std.flatten(), bins=100, range=(0,100) )\n",
+ "ax0.plot([hist_std_tresh, hist_std_tresh], [0, np.nanmax(a[0])], linewidth=1.5, color='red' ) \n",
+ "ax0.set_xlabel('Histogram width [ADU]', fontsize=14)\n",
+ "ax0.set_ylabel('Number of histograms', fontsize=14)\n",
+ "ax0.set_title(f'Module {module}, {hist_std[hist_std<hist_std_tresh].shape[0]} histograms below threshold',\n",
+ " fontsize=14, fontweight='bold')\n",
+ "ax0.grid()\n",
+ "plt.yscale('log')\n",
+ "\n",
+ "\n",
+ "fig = plt.figure(figsize=(10, 5))\n",
+ "ax0 = fig.add_subplot(111)\n",
+ "a = ax0.hist(hist_mean.flatten(), bins=100, range=(0,np.nanmax(hist_mean)) )\n",
+ "ax0.plot([hist_mean_tresh, hist_mean_tresh], [0, np.nanmax(a[0])], linewidth=1.5, color='red' ) \n",
+ "ax0.set_xlabel('Histogram mean [ADU]', fontsize=14)\n",
+ "ax0.set_ylabel('Number of histograms', fontsize=14)\n",
+ "ax0.set_title(f'Module {module}, {hist_mean[hist_mean<hist_mean_tresh].shape[0]} histograms below threshold',\n",
+ " fontsize=16, fontweight='bold')\n",
+ "ax0.grid()\n",
+ "plt.yscale('log')\n",
+ "\n",
+ "\n",
+ "fig = plt.figure(figsize=(10, 5))\n",
+ "ax0 = fig.add_subplot(111)\n",
+ "a = ax0.hist(h_sums.flatten(), bins=100, range=(0,np.max(h_sums)) )\n",
+ "ax0.plot([h_sums_tresh, h_sums_tresh], [0, np.nanmax(a[0])], linewidth=1.5, color='red' ) \n",
+ "ax0.set_xlabel('Number of entries in histogram', fontsize=14)\n",
+ "ax0.set_ylabel('Number of histograms', fontsize=14)\n",
+ "ax0.set_title(f'Module {module}, {h_sums[h_sums<h_sums_tresh].shape[0]} histograms below threshold',\n",
+ " fontsize=16, fontweight='bold')\n",
+ "ax0.grid()\n",
+ "plt.yscale('log')\n",
+ "\n",
+ "fig = plt.figure(figsize=(10, 5))\n",
+ "ax0 = fig.add_subplot(111)\n",
+ "a = ax0.hist(h_out.flatten(), bins=100, range=(-1, 0))\n",
+ "ax0.set_xlabel('Fraction of outliers in histograms', fontsize=14)\n",
+ "ax0.set_ylabel('Number of histograms', fontsize=14)\n",
+ "ax0.set_title(f'Module {module}', fontsize=16, fontweight='bold')\n",
+ "ax0.grid()\n",
+ "plt.yscale('log')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Analysis across cells\n",
+ "\n",
+ "Behavior of histograms across cells is shown below by presenting values averaged across pixels. Mean value of the histogram have quite visible deviation across cells. This deviation may be driven either by the position of photon peak or by different (across cells/pulses) intensity of the beam. Error-band shows deviation of the mean value across pixels.\n",
+ "\n",
+ "Number of entries per histogram averaged across pixels should nearly coincides with number of processed images for given cell. Both lines expected to be flat in case of processing on only one dataset (one run). Zero number of images suggests, that corresponding region of cells was not covered by dataset or not processed.\n",
+ "\n",
+ "Histogram range is chosen to cover noise peak and all visible photon peaks. Significant number of outliers may make photon peak analysis impossible."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "x = np.linspace(0, n_cells, n_cells)\n",
+ "fig = plt.figure(figsize=(10, 12))\n",
+ "\n",
+ "ax0 = fig.add_subplot(211)\n",
+ "ax0.plot(x, hist_mean_pix, 'k', color='#3F7F4C')\n",
+ "ax0.fill_between(x, hist_mean_pix-hist_std_pix, hist_mean_pix+hist_std_pix,\n",
+ " alpha=0.6, edgecolor='#3F7F4C', facecolor='#7EFF99',\n",
+ " linewidth=1, linestyle='dashdot', antialiased=True,\n",
+ " label=\" mean value $ \\pm $ std \")\n",
+ "\n",
+ "ax0.set_xlabel('Cell', fontsize=14)\n",
+ "ax0.set_ylabel('Mean over pixels [ADU]', fontsize=14)\n",
+ "ax0.set_title(f'Module {module}', fontsize=16, fontweight='bold')\n",
+ "ax0.grid()\n",
+ "_ = ax0.legend()\n",
+ "\n",
+ "ax1 = fig.add_subplot(212)\n",
+ "ax1.plot(x, h_sum_mean_pix, 'k', color='#3F7F4C', label='# entries per histograms')\n",
+ "ax1.plot(x, total_img, 'k', color='red', linestyle='dashdot', label='# processed images')\n",
+ "\n",
+ "\n",
+ "\n",
+ "ax1.set_xlabel('Cell', fontsize=14)\n",
+ "ax1.set_ylabel('# entries averaged across pixels', fontsize=14)\n",
+ "ax1.set_title(f'Module {module}', fontsize=16, fontweight='bold')\n",
+ "_ = ax1.legend()\n",
+ "\n",
+ "\n",
+ "fig = plt.figure(figsize=(10, 6))\n",
+ "ax0 = fig.add_subplot(111)\n",
+ "ax0.plot(x, 1-h_sum_mean_pix/total_img, 'k', color='#3F7F4C')\n",
+ "\n",
+ "\n",
+ "ax0.set_xlabel('Cell', fontsize=14)\n",
+ "ax0.set_ylabel('Average fraction of outliers', fontsize=14)\n",
+ "ax0.set_title(f'Module {module}', fontsize=16, fontweight='bold')\n",
+ "ax0.grid()\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Single pixel overview\n",
+ "\n",
+ "Plot below shows histograms for single pixel and all memory cells. It should contain noise peak and well pronounced one or few photon peaks. Estimation of the photon peak position requires to have comparable hight of the noise and first photon peak."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Plot for single pixel and all memory cells.\n",
+ "xpix= 23\n",
+ "ypix= 44\n",
+ "\n",
+ "x = np.arange(h_range[0],h_range[1] , 1)\n",
+ "n,_ = rshist[:,:,xpix,ypix].shape\n",
+ "\n",
+ "colors = mpl.cm.rainbow(np.linspace(0, 1, n))\n",
+ "\n",
+ "\n",
+ "fig = plt.figure(figsize=(10,5))\n",
+ "fig.suptitle(f'Module {module} ', fontsize=14, fontweight='bold')\n",
+ "\n",
+ "ax = fig.add_subplot(111)\n",
+ "fig.subplots_adjust(top=0.85)\n",
+ "ax.set_title(f'single pixel [{xpix},{ypix}], all ({n_cells}) memory cells')\n",
+ "\n",
+ "ax.set_xlabel('Signal [ADU]')\n",
+ "ax.set_ylabel('Counts')\n",
+ "ax.set_xlim(-50,300)\n",
+ "for color, y in zip(colors, rshist[:,:,xpix,ypix]):\n",
+ " ax.plot(x, y, color=color,linewidth=0.2)\n",
+ "plt.grid() \n",
+ "plt.show()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index 0f13d654da6d6b0a0cb59a3f51c46a71229dd2cd..fe16bfb27f8130a3fcb4a7568d2e0b6ef53695e8 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -42,6 +42,13 @@ notebooks = {
"default concurrency": None,
"cluster cores": 8},
},
+ "FF_HISTS": {
+ "notebook":
+ "notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb",
+ "concurrency": {"parameter": "modules",
+ "default concurrency": list(range(16)),
+ "cluster cores": 1},
+ },
},
"AGIPD64K": {
"DARK": {