diff --git a/notebooks/AGIPD/playground/AGIPD_FF_Prepare_data.ipynb b/notebooks/AGIPD/playground/AGIPD_FF_Prepare_data.ipynb
deleted file mode 100644
index 19794465d50e42d9cdc6c6aaa63e0a94d6d49eb5..0000000000000000000000000000000000000000
--- a/notebooks/AGIPD/playground/AGIPD_FF_Prepare_data.ipynb
+++ /dev/null
@@ -1,313 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# AGIPD Analysis\n",
- "\n",
- "Author: European XFEL Detector Group, Version: 1.0\n",
- "\n",
- "Offline Calibration for the AGIPD Detector"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = \"/gpfs/exfel/exp/SPB/202030/p900138/proc/\" # the folder to read data from, required\n",
- "modules = [2] # module to consider, range allowed\n",
- "out_folder = \"/gpfs/exfel/exp/MID/202030/p900137/scratch/karnem/r0319_0322_0342_v02\" # the folder to output to, required\n",
- "cluster_profile = \"noDB\"\n",
- "fname = '{}/CORR-R{:04d}-AGIPD{:02d}-S{:05d}.h5'\n",
- "sequences = [-1] # module to consider, set to -1 for all, range allowed\n",
- "cells = 'range(0,0)' # number of cells, expression should be equivalent to list\n",
- "n_bins = 500 # number of bins of the histogram\n",
- "h_range = [-50, 450] # range of the histogram\n",
- "chunk_size = 5 # Number of memory cells to be processed at ones and be stored in a file\n",
- "n_cells = 202 # total number of memory cells (used to create summary file)\n",
- "run = 204 # run number, required\n",
- "karabo_id = 'MID_DET_AGIPD1M-1' # karabo_id \n",
- "templ = \"{out_folder}/hists_m{module:02d}_c*.h5\" # Template to concatinate histograms"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from functools import partial\n",
- "from ipyparallel import Client\n",
- "import warnings\n",
- "import glob\n",
- "import h5py\n",
- "import numpy as np\n",
- "from time import sleep, time\n",
- "import os\n",
- "import gc\n",
- "import matplotlib.pyplot as plt\n",
- "import matplotlib as mpl\n",
- "from cal_tools.ana_tools import save_dict_to_hdf5\n",
- "%matplotlib inline\n",
- "warnings.filterwarnings('ignore')\n",
- "client = Client(profile=cluster_profile)\n",
- "view = client[:]\n",
- "view.use_dill()\n",
- "\n",
- "in_folder = \"{}/r{:04d}/\".format(in_folder, run)\n",
- "cells = list(eval(cells))\n",
- "if len(cells)>0:\n",
- " print(f'List of cells {cells[0]}-{cells[-1]}: {cells}')\n",
- "print(f'Input folder {in_folder}')\n",
- "n_pix = 128*512\n",
- "module = modules[0]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "scrolled": false
- },
- "outputs": [],
- "source": [
- "def process_file(cells, n_bins, h_range, n_pix, module, karabo_id, file_name):\n",
- " import glob\n",
- " import h5py\n",
- " import numpy as np\n",
- " import gc\n",
- "\n",
- " print('Read file {}'.format(file_name))\n",
- " sequence = int(file_name.split('/')[-1].split('-')[3][1:-3])\n",
- " res_all = np.zeros((len(cells), n_bins, n_pix)).astype(np.uint32)\n",
- " err = ''\n",
- " try:\n",
- " with h5py.File(file_name, \"r\") as f:\n",
- " path = f'/INSTRUMENT/{karabo_id}/DET/{module}CH0:xtdf/image'\n",
- " print(path)\n",
- "\n",
- " data_h = f[f'{path}/data']\n",
- " cellId_h = f[f'{path}/cellId']\n",
- "\n",
- " cell_id = np.array(cellId_h[()])\n",
- " \n",
- " for ic, cell in enumerate(cells):\n",
- " print(cell)\n",
- " cell_sel = np.where(cell_id == cell)\n",
- " data = np.array(data_h[cell_sel]).astype(np.float32)\n",
- "\n",
- " print(data.shape)\n",
- " res_all[ic] = np.apply_along_axis(lambda a: np.histogram(a, bins=n_bins, range=h_range)[0],\n",
- " 0,\n",
- " data.reshape(data.shape[0], n_pix))\n",
- " gc.collect()\n",
- " except Exception as e:\n",
- " err = str(e)\n",
- " gc.collect()\n",
- " return res_all, err, sequence\n",
- "\n",
- "\n",
- "if sequences[0] == -1:\n",
- " fnames = glob.glob(fname.format(\n",
- " in_folder, run, module, 99999).replace('99999', '*'))\n",
- "else:\n",
- " fnames = [fname.format(in_folder, run, module, x) for x in sequences]\n",
- "\n",
- "#proposal = int(in_folder.split('/')[6][1:])\n",
- "sequences = [int(x.split('/')[-1].split('-')[3][1:-3]) for x in fnames]\n",
- "\n",
- "print(f\"List of sequences: {sorted(sequences)}\")\n",
- "processed = np.zeros((max(sequences+[1])+1, max(cells+[1])+1))\n",
- "processed[:,:] = np.nan\n",
- "\n",
- "cell_list = []\n",
- "for cell in cells:\n",
- " cell_list.append(cell)\n",
- " if len(cell_list) >= chunk_size or cell == cells[-1]:\n",
- " inp = []\n",
- " for file_name in fnames:\n",
- " inp.append(file_name)\n",
- "\n",
- " print(f'Process cells: {cell_list}')\n",
- " p = partial(process_file, cell_list, n_bins, h_range, n_pix, module, karabo_id)\n",
- " results = view.map_sync(p, inp)\n",
- " #results = list(map(p, inp))\n",
- "\n",
- " all_hists = np.zeros((len(cell_list), n_bins, n_pix)).astype(np.uint32)\n",
- " for ir, r in enumerate(results):\n",
- " data, msg, s = r\n",
- " if msg == '':\n",
- " processed[s, np.array(cell_list)] = 1\n",
- " all_hists += data\n",
- " else:\n",
- " processed[s, np.array(cell_list)] = 0\n",
- " print(f'Error in {ir}: {msg}')\n",
- "\n",
- "\n",
- " out_name = '{}/hists_m{:02d}_c{:03d}-{:03d}.h5'.format(out_folder, module,\n",
- " cell_list[0], cell_list[-1])\n",
- "\n",
- " save_dict_to_hdf5({'hist': all_hists,\n",
- " 'cellId': np.array(cell_list),\n",
- " # 'proposal': proposal,\n",
- " # 'sequences': sequences,\n",
- " # 'run_': [int(run)],\n",
- " 'nBins': n_bins,\n",
- " 'hRange': np.array(h_range)}, out_name)\n",
- "\n",
- " cell_list = []\n",
- " gc.collect()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "fig = plt.figure(figsize=(10,10))\n",
- "plt.imshow(processed, cmap=plt.cm.jet, vmin = 0, vmax = 1)\n",
- "plt.title(f\"Processed files and cells for module {module}\")\n",
- "plt.ylabel(\"Sequences\", fontsize=18)\n",
- "_ = plt.xlabel(\"Cells\", fontsize=18)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Concatinate all files for given module\n",
- "fnames = glob.glob(templ.format(out_folder=out_folder, module=module))\n",
- "total_hist = np.zeros((n_cells, n_bins, n_pix)).astype(np.uint32)\n",
- "\n",
- "for file_name in fnames:\n",
- " with h5py.File(file_name, \"r\") as f:\n",
- "\n",
- " f_hist = np.array(f['hist'][()])\n",
- " f_cell_id = np.array(f['cellId'][()])\n",
- " f_n_bins = f['nBins'][()]\n",
- " f_h_range = np.array(f['hRange'][()])\n",
- " #f_proposal = np.array(f['proposal'][()])\n",
- " #f_sequences = np.array(f['sequences'][()])\n",
- " #f_runs = np.array(f['runs'][()])\n",
- "\n",
- " if n_bins != f_n_bins or f_h_range[0] != h_range[0] or f_h_range[1] != h_range[1]:\n",
- " print(f'file {file_name} is incompatible to be merged')\n",
- " continue\n",
- "\n",
- " print(f'Add file {file_name} with cells {f_cell_id}')\n",
- " total_hist[f_cell_id] += f_hist\n",
- "\n",
- "\n",
- "out_name = '{}/hists_m{:02d}_sum.h5'.format(out_folder, module)\n",
- "print(f'Save to file: {out_name}')\n",
- "\n",
- "save_dict_to_hdf5({'hist': total_hist,\n",
- " 'cellId': np.arange(n_cells),\n",
- " 'nBins': n_bins,\n",
- " 'hRange': np.array(h_range)}, out_name)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "rshist = np.reshape(total_hist, (n_cells, n_bins, 512, 128))\n",
- "\n",
- "# some sanity check per mem cell\n",
- "mean_hist = np.zeros(n_cells)\n",
- "std_hist = np.zeros(n_cells)\n",
- "sum_hist = np.zeros(n_cells)\n",
- "for i in range(0, n_cells):\n",
- " mean_hist[i] = np.mean(rshist[i, :, :, :])\n",
- " std_hist[i] = np.std(rshist[i, :, :, :])\n",
- " sum_hist[i] = np.sum(rshist[i, :, :, :])/(128*512)\n",
- "\n",
- "x = np.linspace(0, n_cells, n_cells)\n",
- "fig = plt.figure(figsize=(10, 10))\n",
- "\n",
- "ax0 = fig.add_subplot(211)\n",
- "ax0.plot(x, mean_hist, 'k', color='#3F7F4C')\n",
- "ax0.fill_between(x, mean_hist-std_hist, mean_hist+std_hist,\n",
- " alpha=0.6, edgecolor='#3F7F4C', facecolor='#7EFF99',\n",
- " linewidth=1, linestyle='dashdot', antialiased=True,\n",
- " label=\" mean value $ \\pm $ std \")\n",
- "\n",
- "ax0.set_xlabel('Cell', fontsize=14)\n",
- "ax0.set_ylabel('Mean over module [ADU]', fontsize=14)\n",
- "ax0.set_title(f'Module {module}', fontsize=16, fontweight='bold')\n",
- "ax0.grid()\n",
- "# ax0.set_ylim(-100,100)\n",
- "_ = ax0.legend()\n",
- "\n",
- "ax1 = fig.add_subplot(212)\n",
- "ax1.plot(x, sum_hist, 'k', color='#3F7F4C')\n",
- "\n",
- "ax1.set_xlabel('Cell', fontsize=14)\n",
- "ax1.set_ylabel('Average statistics', fontsize=14)\n",
- "ax1.set_title(f'Module {module}', fontsize=16, fontweight='bold')\n",
- "_ = ax1.legend()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Plot for single pixel and all memory cells.\n",
- "xpix= 23\n",
- "ypix= 44\n",
- "\n",
- "x = np.arange(h_range[0],h_range[1] , 1)\n",
- "n,_ = rshist[:,:,xpix,ypix].shape\n",
- "\n",
- "colors = mpl.cm.rainbow(np.linspace(0, 1, n))\n",
- "\n",
- "\n",
- "fig = plt.figure(figsize=(10,5))\n",
- "fig.suptitle(f'Module {module} ', fontsize=14, fontweight='bold')\n",
- "\n",
- "ax = fig.add_subplot(111)\n",
- "fig.subplots_adjust(top=0.85)\n",
- "ax.set_title(f'single pixel [{xpix},{ypix}], all ({n_cells}) memory cells')\n",
- "\n",
- "ax.set_xlabel('Signal [ADU]')\n",
- "ax.set_ylabel('Counts')\n",
- "ax.set_xlim(-50,300)\n",
- "for color, y in zip(colors, rshist[:,:,xpix,ypix]):\n",
- " ax.plot(x, y, color=color,linewidth=0.2)\n",
- "plt.grid() \n",
- "plt.show()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.7"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index c3a19cdd95c825eff0af0968812fe5b51f05671e..b814077b51b6e0d5c3913795105024b9887a0143 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -41,12 +41,6 @@ notebooks = {
"default concurrency": None,
"cluster cores": 8},
},
- "FF_HISTS": {
- "notebook": "notebooks/AGIPD/playground/AGIPD_FF_Prepare_data.ipynb",
- "concurrency": {"parameter": "modules",
- "default concurrency": list(range(16)),
- "cluster cores": 30},
- },
},
"AGIPD64K": {
@@ -241,4 +235,3 @@ notebooks = {
},
},
}
-