diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 5fbf7f6a6abe8f48dcd996fd41ce99ae3e482614..d76e54a596edc181762bc808544d6158606d7de3 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -1345,9 +1345,9 @@ class AgipdCorrections:
connect to
* tcp://host:port_low#port_high to specify a port range from
which a random port will be picked. E.g. specifying
-
+
tcp://max-exfl016:8015#8025
-
+
will randomly pick an address in the range max-exfl016:8015 and
max-exfl016:8025.
@@ -1444,48 +1444,55 @@ class AgipdCorrections:
# [prot, serv, str(np.random.randint(int(r1), int(r2)))])
dinstance = getattr(Detectors, self.cal_det_instance)
- offset, when = get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.Offset(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db_dark,
- bias_voltage=bias_voltage,
- acquisition_rate=self.acquisition_rate),
- np.zeros((128, 512,
- max_cells_db,
- 3)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
-
- noise = get_constant_from_db(getattr(dinstance, qm),
- Constants.AGIPD.Noise(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db_dark,
- bias_voltage=bias_voltage,
- acquisition_rate=self.acquisition_rate),
- np.zeros((128, 512, max_cells_db, 3)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
-
- bpixels = get_constant_from_db(getattr(dinstance, qm),
- Constants.AGIPD.BadPixelsDark(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db_dark,
- bias_voltage=bias_voltage,
- acquisition_rate=self.acquisition_rate),
- np.zeros((128, 512, max_cells_db, 3)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db).astype(
- np.uint32)
-
- thresholds = get_constant_from_db(getattr(dinstance, qm),
+ when = {}
+ offset, when['offset'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.Offset(),
+ Conditions.Dark.AGIPD(
+ memory_cells=max_cells_db_dark,
+ bias_voltage=bias_voltage,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.zeros((128, 512,
+ max_cells_db,
+ 3)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
+
+ noise, when['noise'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.Noise(),
+ Conditions.Dark.AGIPD(
+ memory_cells=max_cells_db_dark,
+ bias_voltage=bias_voltage,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.zeros(
+ (128, 512, max_cells_db, 3)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
+
+ bpixels, when['bpixels'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.BadPixelsDark(),
+ Conditions.Dark.AGIPD(
+ memory_cells=max_cells_db_dark,
+ bias_voltage=bias_voltage,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.zeros(
+ (128, 512, max_cells_db, 3)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
+ bpixels = bpixels.astype(np.uint32)
+
+ thresholds, when['thresholds'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
Constants.AGIPD.ThresholdsDark(),
Conditions.Dark.AGIPD(
memory_cells=max_cells_db_dark,
bias_voltage=bias_voltage,
- acquisition_rate=self.acquisition_rate),
+ acquisition_rate=self.acquisition_rate), # noqa
np.ones((128, 512, max_cells_db, 2)),
cal_db_interface,
creation_time=creation_time,
@@ -1499,39 +1506,44 @@ class AgipdCorrections:
return when
# load also non-dark constants from the database
- slopesPC = get_constant_from_db(getattr(dinstance, qm),
- Constants.AGIPD.SlopesPC(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db,
- bias_voltage=bias_voltage,
- acquisition_rate=self.acquisition_rate),
- np.ones((128, 512, max_cells_db, 10)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
+ slopesPC, when['slopesPC'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.SlopesPC(),
+ Conditions.Dark.AGIPD(
+ memory_cells=max_cells_db,
+ bias_voltage=bias_voltage,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.ones(
+ (128, 512, max_cells_db, 10)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
slopesPC = slopesPC.astype(np.float32)
# this will handle some historical data in a slighly different format
- if slopesPC.shape[0] == 10: # constant dimension injected first
+ # constant dimension injected first
+ if slopesPC.shape[0] == 10 or slopesPC.shape[0] == 11:
slopesPC = np.moveaxis(slopesPC, 0, 3)
slopesPC = np.moveaxis(slopesPC, 0, 2)
- slopesFF = np.squeeze(
- get_constant_from_db(getattr(dinstance, qm),
- Constants.AGIPD.SlopesFF(),
- Conditions.Illuminated.AGIPD(max_cells_db,
- bias_voltage,
- photon_energy,
- pixels_x=512,
- pixels_y=128,
- beam_energy=None,
- acquisition_rate=self.acquisition_rate),
- # noqa
- np.ones((128, 512, max_cells_db, 2)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db))
+ slopesFF, when['slopesFF'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.SlopesFF(),
+ Conditions.Illuminated.AGIPD(
+ max_cells_db,
+ bias_voltage,
+ photon_energy,
+ pixels_x=512,
+ pixels_y=128,
+ beam_energy=None,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.ones((128, 512, max_cells_db, 2)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
+ slopesFF = np.squeeze(slopesFF)
+ print(slopesFF.shape, )
# calculate relative gain from the constants
# we default to a value of one, so basically a unity transform
rel_gain = np.ones((128, 512, self.max_cells, 3), np.float32)
@@ -1588,31 +1600,35 @@ class AgipdCorrections:
slopesPC[..., :self.max_cells, 4], xrayOffset]
# add additonal bad pixel information
- bppc = get_constant_from_db(getattr(Detectors.AGIPD1M1, qm),
- Constants.AGIPD.BadPixelsPC(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db,
- bias_voltage=bias_voltage),
- np.zeros((max_cells_db, 128, 512)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
+ bppc, when['bppc'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.BadPixelsPC(),
+ Conditions.Dark.AGIPD(
+ memory_cells=max_cells_db,
+ bias_voltage=bias_voltage,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.zeros((max_cells_db, 128, 512)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
bppc = np.moveaxis(bppc.astype(np.uint32), 0, 2)
bpixels |= bppc[..., :bpixels.shape[2], None]
- bpff = get_constant_from_db(getattr(Detectors.AGIPD1M1, qm),
- Constants.AGIPD.BadPixelsFF(),
- Conditions.Illuminated.AGIPD(max_cells_db,
- bias_voltage,
- photon_energy,
- pixels_x=512,
- pixels_y=128,
- beam_energy=None),
- # noqa
- np.zeros((128, 512, max_cells_db)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
+ bpff, when['bpff'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.BadPixelsFF(),
+ Conditions.Illuminated.AGIPD(
+ max_cells_db,
+ bias_voltage,
+ photon_energy,
+ pixels_x=512,
+ pixels_y=128,
+ beam_energy=None,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.zeros((128, 512, max_cells_db)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
bpixels |= bpff.astype(np.uint32)[..., :bpixels.shape[2], None]
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 6d015baab93ebdcc3307e76e6161619673195f5a..af85abf7dfe201047a54d2d06a46d790dea57564 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -22,9 +22,9 @@
},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/MID/201931/p900090/raw/\" # the folder to read data from, required\n",
- "run = 5 # runs to process, required\n",
- "out_folder = \"/gpfs/exfel/exp/MID/201931/p900090/proc/\" # the folder to output to, required\n",
+ "in_folder = \"/gpfs/exfel/exp/SPB/201901/p002545/raw\" # the folder to read data from, required\n",
+ "run = 9 # runs to process, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/AGIPD_Corr12\" # the folder to output to, required\n",
"calfile = \"/gpfs/exfel/data/scratch/haufs/agipd_on_demand/agipd_store_mid.h5\" # path to calibration file. Leave empty if all data should come from DB\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
@@ -339,7 +339,8 @@
"ExecuteTime": {
"end_time": "2019-02-21T11:30:16.057429Z",
"start_time": "2019-02-21T11:30:10.082114Z"
- }
+ },
+ "scrolled": false
},
"outputs": [],
"source": [
@@ -468,11 +469,9 @@
" print(\"Initialized constants\")\n",
" \n",
" for irange in agipd_corr.get_iteration_range():\n",
- " \n",
" agipd_corr.correct_agipd(irange)\n",
" print(\"Iterated\")\n",
- " \n",
- " \n",
+ "\n",
" print(\"All interations finished\")\n",
" hists, edges = agipd_corr.get_histograms()\n",
" hists_signal_low, hists_signal_high, hists_gain_vs_signal, hists_dig_gain_vs_signal= hists\n",
@@ -497,7 +496,7 @@
" #influx = create_influx_entry(run, proposal, qm, sequence, filesize, CHUNK_SIZE, total_sequences, success, duration, reason)\n",
" #client.write_points([influx])\n",
" return (hists_signal_low, hists_signal_high, hists_gain_vs_signal, hists_dig_gain_vs_signal,\n",
- " low_edges, high_edges, signal_edges, dig_signal_edges, gain_stats, max_cells, when, err)\n",
+ " low_edges, high_edges, signal_edges, dig_signal_edges, gain_stats, max_cells, when, qm, err)\n",
" \n",
"done = False\n",
"first_files = []\n",
@@ -556,9 +555,9 @@
"\n",
" for rr in r:\n",
" if rr is not None:\n",
- " hl, hh, hg, hdg, low_edges, high_edges, signal_edges, dig_signal_edges, gs, cells, when, err = rr\n",
+ " hl, hh, hg, hdg, low_edges, high_edges, signal_edges, dig_signal_edges, gs, cells, when, qm, err = rr\n",
" all_cells.append(cells)\n",
- " whens.append(when)\n",
+ " whens.append((qm, when))\n",
" errors.append(err)\n",
" if hl is not None: # any one being None will also make the others None\n",
" hists_signal_low += hl.astype(np.float64)\n",
@@ -576,14 +575,38 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
- "print(\"Offset where injected on: \")\n",
- "for i, when in enumerate(whens):\n",
- " qm = \"Q{}M{}\".format((i%16)//4 + 1, i%4 +1)\n",
- " if errors[i] is None: \n",
- " print(\"{}: {} \".format(qm, when))\n",
+ "print(\"Constants were injected on: \")\n",
+ "to_store = []\n",
+ "for i, (qm, when) in enumerate(whens):\n",
+ " print(qm)\n",
+ " line = [qm]\n",
+ " # If correction is crashed\n",
+ " if errors[i] is not None:\n",
+ " print(\"Error: {}\".format(errors[i]) )\n",
" else:\n",
- " print(\"{}: {}. Reason: {}\".format(qm, when, errors[i]))"
+ " for key, item in when.items():\n",
+ " if hasattr(item, 'strftime'):\n",
+ " item = item.strftime('%y-%m-%d %H:%M')\n",
+ " # If constant retrieval is crashed\n",
+ " else:\n",
+ " item = 'None'\n",
+ " when[key] = item\n",
+ " print('{:.<12s}'.format(key), item)\n",
+ " \n",
+ " # Store few time stamps if exists\n",
+ " # Add NA to keep array structure\n",
+ " for key in ['offset', 'slopesPC', 'slopesFF']:\n",
+ " if when and key in when:\n",
+ " line.append(when[key])\n",
+ " else:\n",
+ " if errors[i] is not None:\n",
+ " line.append('Err')\n",
+ " else:\n",
+ " line.append('NA')\n",
+ " to_store.append(line)\n",
+ "\n",
+ "np.savetxt(\"{}/tmp_const_beginat_S{:05d}.csv\".format(out_folder, sequences[0]), \n",
+ " np.array(to_store).astype(str), fmt='%s', delimiter = ',')"
]
},
{
@@ -788,7 +811,7 @@
" gains.append(np.zeros((last_idx-first_idx, 512, 128)))\n",
" mask.append(np.zeros((last_idx-first_idx, 512, 128)))\n",
" raw.append(np.zeros((last_idx-first_idx, 512, 128)))\n",
- "\n"
+ " "
]
},
{
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..eed58a38801a79ee08b768103d17d3b50fcb3b93
--- /dev/null
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
@@ -0,0 +1,127 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Summary of the AGIPD offline correction #"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cluster_profile = \"noDB\" # The ipcluster profile to use\n",
+ "run = 555 # runs to process, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/AGIPD_Bla3\" # path to output to, required"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import dateutil.parser\n",
+ "import glob\n",
+ "import numpy as np\n",
+ "import re\n",
+ "import os\n",
+ "import warnings\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "import matplotlib.pyplot as plt\n",
+ "%matplotlib inline\n",
+ "import tabulate\n",
+ "from IPython.display import display, Markdown, Latex"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# load temporary data from csv files\n",
+ "fnames = sorted(glob.glob('{}/r{:04d}/tmp_const_beginat_*'.format(out_folder,run)))\n",
+ "\n",
+ "const_times = []\n",
+ "seq = []\n",
+ "for fname in fnames:\n",
+ " # read sequence number and array of timestamps\n",
+ " s = int(re.findall(r'.*S([0-9]{5}).*', fname)[0])\n",
+ " arr = np.loadtxt(fname, dtype=str, delimiter = ',')\n",
+ " # job can run across several sequences\n",
+ " # number of modules is fixed to 16\n",
+ " if arr.shape[0]>16:\n",
+ " seq += range(int(s), int(s)+arr.shape[0]//16)\n",
+ " arr = arr.reshape(arr.shape[0]//16, 16, arr.shape[1])\n",
+ " const_times = const_times + list(arr)\n",
+ " \n",
+ " else:\n",
+ " const_times.append(arr)\n",
+ " seq.append(s)\n",
+ " os.remove(fname)\n",
+ "\n",
+ "const_times = np.array(const_times)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Function print summary of constant injection time\n",
+ "# To reduce printouts only unique entries are shown.\n",
+ "def plot_const_table(const, pos):\n",
+ " print(\"{} were injected on: \".format(const))\n",
+ "\n",
+ " unique, idx, counts = np.unique(const_times[:,:,pos], return_inverse=True, return_counts=True)\n",
+ " idx = idx.reshape((const_times.shape[0],16))\n",
+ "\n",
+ " table = [ [const_times[:,:,pos][idx==0][0] , 'All modules'] ]\n",
+ "\n",
+ " for i in range(1, counts.shape[0]):\n",
+ " line = [ const_times[:,:,pos][idx==i][0] ]\n",
+ " mods = ''\n",
+ " for i_s, s in enumerate(seq):\n",
+ " if( const_times[i_s,:,0][idx[i_s]==i].shape[0]>0 ):\n",
+ " table[0][1] = 'Most of the modules'\n",
+ " mods = mods+ 'S{}: {}, '.format(seq[i_s], const_times[i_s,:,0][idx[i_s]==i])\n",
+ " line.append(mods)\n",
+ " table.append(line)\n",
+ "\n",
+ " md = display(Latex(tabulate.tabulate(table, tablefmt='latex', \n",
+ " headers=[\"Time stamps\", \"Modules and sequences\"]))) \n",
+ " \n",
+ "for i_key, key in enumerate(['offset', 'slopesPC', 'slopesFF']):\n",
+ " if const_times.shape[2]>i_key+1:\n",
+ " plot_const_table(key, i_key+1)\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index a004bdb0e2bc30d95687a032f0e8f6d1c9ca6c02..d9b14de843db5de4c04d84e4f11c187d9d8e44b7 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -22,6 +22,7 @@ notebooks = {
},
"CORRECT": {
"notebook": "notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb",
+ "dep_notebooks": ["notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb"],
"concurrency": {"parameter": "sequences",
"use function": "balance_sequences",
"default concurrency": [-1],