From be31dd794bcc09c3a724a41ca770dced2fbda59a Mon Sep 17 00:00:00 2001
From: Karim Ahmed <karim.ahmed@xfel.eu>
Date: Mon, 28 Oct 2019 15:30:12 +0100
Subject: [PATCH] MR comments and fixing histogram
---
cal_tools/cal_tools/agipdlib.py | 73 ++++++++++---------
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 45 +++++-------
2 files changed, 58 insertions(+), 60 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 359c1ba66..b7a482f51 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -72,8 +72,8 @@ class AgipdCorrections:
"""
def __init__(self, infile, outfile, max_cells, channel, max_pulses,
- rng_pulse_idx, bins_gain_vs_signal,
- bins_signal_low_range, bins_signal_high_range,
+ bins_gain_vs_signal, bins_signal_low_range,
+ bins_signal_high_range,
bins_dig_gain_vs_signal, raw_fmt_version=2, chunk_size=512,
h5_data_path="INSTRUMENT/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/",
h5_index_path="INDEX/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/",
@@ -91,8 +91,8 @@ class AgipdCorrections:
:param max_cell: maximum number of memory cells to handle, e.g. if
calibration constants only exist for a subset of cells
:param channel: module/channel to correct
- :param max_pulses: maximum pulse indices to consider for preview histograms
- :param rng_pulse_idx: range of pulse indices to consider for preview histograms
+ :param max_pulses: a range list of pulse indices used for
+ calibration and histogram. [start, end, step]
:param bins_gain_vs_signal: number of bins for gain vs signal histogram
:param bins_signal_low_range: number of bins for the low signal
range histogram
@@ -121,9 +121,11 @@ class AgipdCorrections:
self.index_v = raw_fmt_version
self.chunksize = chunk_size
self.initialized = False
- self.max_pulses = max_pulses
- self.rng_pulse_idx = rng_pulse_idx
+ self.pulses_lst = list(range(*max_pulses))
+ self.min_pulse = self.pulses_lst[0]
+ self.max_pulse = self.pulses_lst[-1]
self.max_cells = max_cells
+ self.hist_pulses = 0
self.hists_signal_low = 0
self.hists_signal_high = 0
self.hists_gain_vs_signal = 0
@@ -1029,25 +1031,23 @@ class AgipdCorrections:
if self.melt_snow is not False:
msk |= snowmask
- agipd_base = self.agipd_base
- allpulses = np.squeeze(self.infile[agipd_base + "image/pulseId"])
-
# for the first chunk output some statistics
if cidx == 0:
copim = copy.copy(im)
copim[copim < self.median_noise] = np.nan
- bins = (self.bins_signal_low_range, self.max_pulses)
- rnge = [[-50, 1000], [self.pulse_hist_first,
- self.pulse_hist_last + 1]]
+ bins = (self.bins_signal_low_range, self.max_pulse)
+ rnge = [[-50, 1000], [self.min_pulse,
+ self.max_pulse + 1]]
H, xe, ye = np.histogram2d(np.nanmean(copim, axis=(1, 2)),
pulseId,
bins=bins,
range=rnge)
+ self.hist_pulses = self.max_pulse
self.hists_signal_low += H
self.low_edges = (xe, ye)
- bins = (self.bins_signal_high_range, self.max_pulses)
- rnge = [[0, 200000], [self.pulse_hist_first,
- self.pulse_hist_last + 1]]
+ bins = (self.bins_signal_high_range, self.max_pulse)
+ rnge = [[0, 200000], [self.min_pulse,
+ self.max_pulse + 1]]
H, xe, ye = np.histogram2d(np.nanmean(copim, axis=(1, 2)),
pulseId,
bins=bins,
@@ -1164,33 +1164,40 @@ class AgipdCorrections:
single_image = self.infile[agipd_base + "image/data"][first_index, ...]
single_image = np.array(single_image)
- can_calibrate = (allcells < max_cells)
+ # Calculate the pulse step from the chosen max_pulse range
+ # This is not calculated from max_pulses[2],
+ # as it might not be available.
+ pulse_step = self.pulses_lst[1] - self.pulses_lst[0]
# Make sure the range max doesn't have non valid idx.
- if self.rng_pulse_idx[-1] > last_index-1:
- self.pulse_idx_last = last_index-1
+ if self.pulses_lst[-1] + pulse_step > last_index-1:
+ last_pulse = last_index-1
else:
- self.pulse_idx_last = self.rng_pulse_idx[-1]
-
- if self.rng_pulse_idx[0] >= self.pulse_idx_last:
- self.pulse_idx_first = self.rng_pulse_idx[0] - 1
+ last_pulse = self.pulses_lst[-1] + pulse_step
+ # Check if 1st pulse idx was out of valid range.
+ # If that is the case only calibrate the last step pulse.
+ if self.pulses_lst[0] >= last_pulse:
+ first_pulse = self.pulses_lst[0] - pulse_step
else:
- self.pulse_idx_first = self.rng_pulse_idx[0]
+ first_pulse = self.pulses_lst[0]
+
+ # collect the pulses to be calibrated
+ cal_pulses = allpulses[first_pulse:last_pulse:pulse_step]
- pulse_idx_cal = allpulses[self.pulse_idx_first:
- self.pulse_idx_last+1]
- self.pulse_hist_first = np.min(pulse_idx_cal)
- self.pulse_hist_last = np.max(pulse_idx_cal)
+ # These parameters required also for histogram pulse range.
+ self.min_pulse = np.min(cal_pulses)
+ self.max_pulse = np.max(cal_pulses)
- pulse_step = self.rng_pulse_idx[1] - self.rng_pulse_idx[0]
+ can_calibrate = (allcells < max_cells)
# Check if a specific pulses need to be calibrated
# or with only simple pulse ranging.
- if self.pulse_idx_first > max_cells or pulse_step > 1 or True:
- can_calibrate &= np.isin(allpulses, pulse_idx_cal)
+ if pulse_step > 1 or \
+ allpulses[first_pulse] >= allpulses[last_pulse]:
+ can_calibrate &= np.isin(allpulses, cal_pulses)
else:
- can_calibrate &= (allpulses <= np.max(pulse_idx_cal))
- can_calibrate &= (allpulses >= np.min(pulse_idx_cal))
+ can_calibrate &= (allpulses <= self.max_pulse)
+ can_calibrate &= (allpulses >= self.min_pulse)
if np.count_nonzero(can_calibrate) == 0:
return
@@ -1341,7 +1348,7 @@ class AgipdCorrections:
""" Return preview histograms computed from the first chunk
"""
return ((self.hists_signal_low, self.hists_signal_high,
- self.hists_gain_vs_signal, self.hists_dig_gain_vs_signal),
+ self.hists_gain_vs_signal, self.hists_dig_gain_vs_signal, self.hist_pulses),
(self.low_edges, self.high_edges, self.signal_edges,
self.dig_signal_edges))
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 3f0c2b161..8ca55e0c4 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -32,7 +32,7 @@
"overwrite = True # set to True if existing data should be overwritten\n",
"no_relative_gain = False # do not do relative gain correction\n",
"cluster_profile = \"noDB\"\n",
- "max_pulses = [0, 500, 1] # range list [st, end, step] of maximum pulse indices for the report histogram. 3 allowed maximum list input elements. \n",
+ "max_pulses = [0, 500, 1] # range list [st, end, step] of maximum pulse indices. 3 allowed maximum list input elements. \n",
"local_input = False\n",
"bias_voltage = 300\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
@@ -348,7 +348,7 @@
"from functools import partial\n",
"def correct_module(max_cells, do_rel_gain, index_v, CHUNK_SIZE, total_sequences, sequences_qm, \n",
" bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range,\n",
- " bins_dig_gain_vs_signal, max_pulses, rng_pulse_idx, dbparms, fileparms, nodb, chunk_size_idim,\n",
+ " bins_dig_gain_vs_signal, max_pulses, dbparms, fileparms, nodb, chunk_size_idim,\n",
" special_opts, il_mode, loc, dinstance, force_hg_if_below, force_mg_if_below,\n",
" mask_noisy_adc, adjust_mg_baseline, acq_rate, dont_zero_nans, dont_zero_orange,\n",
" inp):\n",
@@ -399,6 +399,7 @@
" hists_signal_high = None\n",
" hists_gain_vs_signal = None\n",
" hists_dig_gain_vs_signal = None\n",
+ " hist_pulses = None\n",
" low_edges = None\n",
" high_edges = None\n",
" signal_edges = None\n",
@@ -438,7 +439,7 @@
" infile = h5py.File(filename, \"r\", driver=\"core\")\n",
" outfile = h5py.File(filename_out, \"w\")\n",
" \n",
- " agipd_corr = AgipdCorrections(infile, outfile, max_cells, channel, max_pulses, rng_pulse_idx,\n",
+ " agipd_corr = AgipdCorrections(infile, outfile, max_cells, channel, max_pulses,\n",
" bins_gain_vs_signal, bins_signal_low_range,\n",
" bins_signal_high_range, bins_dig_gain_vs_signal,\n",
" do_rel_gain=do_rel_gain, chunk_size_idim=chunk_size_idim,\n",
@@ -473,7 +474,7 @@
"\n",
" print(\"All iterations are finished\")\n",
" hists, edges = agipd_corr.get_histograms()\n",
- " hists_signal_low, hists_signal_high, hists_gain_vs_signal, hists_dig_gain_vs_signal= hists\n",
+ " hists_signal_low, hists_signal_high, hists_gain_vs_signal, hists_dig_gain_vs_signal, hist_pulses = hists\n",
" low_edges, high_edges, signal_edges, dig_signal_edges = edges\n",
" gain_stats = np.array(agipd_corr.gain_stats)\n",
" outfile.close()\n",
@@ -494,7 +495,7 @@
" duration = (datetime.now()-start).total_seconds()\n",
" #influx = create_influx_entry(run, proposal, qm, sequence, filesize, CHUNK_SIZE, total_sequences, success, duration, reason)\n",
" #client.write_points([influx])\n",
- " return (hists_signal_low, hists_signal_high, hists_gain_vs_signal, hists_dig_gain_vs_signal,\n",
+ " return (hists_signal_low, hists_signal_high, hists_gain_vs_signal, hists_dig_gain_vs_signal, hist_pulses,\n",
" low_edges, high_edges, signal_edges, dig_signal_edges, gain_stats, max_cells, when, qm, err)\n",
" \n",
"done = False\n",
@@ -502,31 +503,16 @@
"inp = []\n",
"left = total_sequences\n",
"\n",
- "# Checking the max_pulses input\n",
- "# Applying default values if input not 3 elements.\n",
- "if len(max_pulses) == 3:\n",
- " rng_pulse_idx = list(range(max_pulses[0], max_pulses[1], max_pulses[2]))\n",
- " max_pulses = len(rng_pulse_idx)\n",
- "elif len(max_pulses) == 2:\n",
- " rng_pulse_idx = list(range(max_pulses[0], max_pulses[1], 1))\n",
- " max_pulses = len(rng_pulse_idx)\n",
- "elif len(max_pulses) == 1:\n",
- " rng_pulse_idx = list(range(0, max_pulses[0], 1))\n",
- " max_pulses = len(rng_pulse_idx)\n",
- "else:\n",
- " raise ValueError(\"Wrong input for max_pulses {} \\n\".format(max_pulses),\n",
- " \"max_pulses should have 3 arguments maximum\")\n",
- " \n",
+ "pulses_lst = list(range(*max_pulses))\n",
"print(\"A range of {} pulse indices is selected: from {} to {} with a step of {}\"\n",
- " .format(max_pulses, rng_pulse_idx[0], rng_pulse_idx[-1],\n",
- " rng_pulse_idx[1]-rng_pulse_idx[0]))\n",
+ " .format(len(pulses_lst), pulses_lst[0] , pulses_lst[-1] + (pulses_lst[1] - pulses_lst[0]),\n",
+ " pulses_lst[1] - pulses_lst[0]))\n",
"\n",
"bins_gain_vs_signal = (100, 100)\n",
"bins_signal_low_range = 100\n",
"bins_signal_high_range = 100\n",
"bins_dig_gain_vs_signal = (100, 4)\n",
- "hists_signal_low = np.zeros((bins_signal_low_range, max_pulses), np.float64)\n",
- "hists_signal_high = np.zeros((bins_signal_low_range, max_pulses), np.float64)\n",
+ "\n",
"hists_gain_vs_signal = np.zeros((bins_gain_vs_signal), np.float64)\n",
"hists_dig_gain_vs_signal = np.zeros((bins_dig_gain_vs_signal), np.float64)\n",
"gain_stats = 0\n",
@@ -560,7 +546,7 @@
" print(\"Running {} tasks parallel\".format(len(inp)))\n",
" p = partial(correct_module, max_cells, do_rel_gain, index_v, CHUNK_SIZE, total_sequences,\n",
" sequences_qm, bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range,\n",
- " bins_dig_gain_vs_signal, max_pulses, rng_pulse_idx, dbparms, fileparms, nodb, chunk_size_idim,\n",
+ " bins_dig_gain_vs_signal, max_pulses, dbparms, fileparms, nodb, chunk_size_idim,\n",
" special_opts, il_mode, loc, dinstance, force_hg_if_below, force_mg_if_below,\n",
" mask_noisy_adc, adjust_mg_baseline, acq_rate, dont_zero_nans, dont_zero_orange)\n",
" \n",
@@ -571,12 +557,17 @@
" inp = []\n",
" left -= MAX_PAR\n",
"\n",
+ " init_hist = False\n",
" for rr in r:\n",
" if rr is not None:\n",
- " hl, hh, hg, hdg, low_edges, high_edges, signal_edges, dig_signal_edges, gs, cells, when, qm, err = rr\n",
+ " hl, hh, hg, hdg, hp, low_edges, high_edges, signal_edges, dig_signal_edges, gs, cells, when, qm, err = rr\n",
" all_cells.append(cells)\n",
" whens.append((qm, when))\n",
" errors.append(err)\n",
+ " if not init_hist:\n",
+ " hists_signal_low = np.zeros((bins_signal_low_range, hp), np.float64)\n",
+ " hists_signal_high = np.zeros((bins_signal_low_range, hp), np.float64)\n",
+ " init_hist = True\n",
" if hl is not None: # any one being None will also make the others None\n",
" hists_signal_low += hl.astype(np.float64)\n",
" hists_signal_high += hh.astype(np.float64)\n",
@@ -629,7 +620,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-02-18T17:28:51.765030Z",
--
GitLab