diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 455ee524d72d0fd571d4aab384d9b48cac115a64..84bc94255fcee20b0680116306c45032a9c5fac7 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -15,6 +15,8 @@ def get_num_cells(fname, loc, module):
with h5py.File(fname, "r") as f:
cells = \
f["INSTRUMENT/{}/DET/{}CH0:xtdf/image/cellId".format(loc, module)][()]
+ if cells.shape[0] == 0:
+ return None
maxcell = np.max(cells)
options = [4, 32, 64, 76, 128, 176, 202, 250]
dists = [abs(o - maxcell) for o in options]
@@ -23,10 +25,13 @@ def get_num_cells(fname, loc, module):
def get_acq_rate(fname, loc, module):
with h5py.File(fname, "r") as f:
- pulses = \
- np.squeeze(f["INSTRUMENT/{}/DET/{}CH0:xtdf/image/pulseId".format(
- loc, module)][:2])
- diff = pulses[1] - pulses[0]
+ try:
+ pulses = \
+ np.squeeze(f["INSTRUMENT/{}/DET/{}CH0:xtdf/image/pulseId".format(
+ loc, module)][:2])
+ diff = pulses[1] - pulses[0]
+ except:
+ diff = 0
options = {8: 0.5, 4: 1.1, 2: 2.2, 1: 4.5}
return options.get(diff, None)
@@ -262,7 +267,9 @@ class AgipdCorrections:
med = np.nanmedian(ado, axis=(1, 2))[:, None, None]
doff[:, i * 64:(i + 1) * 64, j * 64:(j + 1) * 64] = med
- self.offset[..., 1] += doff
+ # This is deprecated part of code, which is not compatible with
+ # existing PC constants.
+ # self.offset[..., 1] += doff
if self.adjust_mg_baseline and slopesPC is not None:
x = np.linspace(0, 1000, 1000)
@@ -1157,7 +1164,7 @@ class AgipdCorrections:
if np.count_nonzero(can_calibrate) == 0:
return
allcells = allcells[can_calibrate]
- if self.valid_indices is not None:
+ if self.valid_indices is None:
firange = np.arange(first_index, last_index)
else:
firange = self.valid_indices
@@ -1340,9 +1347,9 @@ class AgipdCorrections:
connect to
* tcp://host:port_low#port_high to specify a port range from
which a random port will be picked. E.g. specifying
-
+
tcp://max-exfl016:8015#8025
-
+
will randomly pick an address in the range max-exfl016:8015 and
max-exfl016:8025.
@@ -1439,48 +1446,55 @@ class AgipdCorrections:
# [prot, serv, str(np.random.randint(int(r1), int(r2)))])
dinstance = getattr(Detectors, self.cal_det_instance)
- offset, when = get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.Offset(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db_dark,
- bias_voltage=bias_voltage,
- acquisition_rate=self.acquisition_rate),
- np.zeros((128, 512,
- max_cells_db,
- 3)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
-
- noise = get_constant_from_db(getattr(dinstance, qm),
- Constants.AGIPD.Noise(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db_dark,
- bias_voltage=bias_voltage,
- acquisition_rate=self.acquisition_rate),
- np.zeros((128, 512, max_cells_db, 3)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
-
- bpixels = get_constant_from_db(getattr(dinstance, qm),
- Constants.AGIPD.BadPixelsDark(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db_dark,
- bias_voltage=bias_voltage,
- acquisition_rate=self.acquisition_rate),
- np.zeros((128, 512, max_cells_db, 3)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db).astype(
- np.uint32)
-
- thresholds = get_constant_from_db(getattr(dinstance, qm),
+ when = {}
+ offset, when['offset'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.Offset(),
+ Conditions.Dark.AGIPD(
+ memory_cells=max_cells_db_dark,
+ bias_voltage=bias_voltage,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.zeros((128, 512,
+ max_cells_db,
+ 3)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
+
+ noise, when['noise'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.Noise(),
+ Conditions.Dark.AGIPD(
+ memory_cells=max_cells_db_dark,
+ bias_voltage=bias_voltage,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.zeros(
+ (128, 512, max_cells_db, 3)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
+
+ bpixels, when['bpixels'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.BadPixelsDark(),
+ Conditions.Dark.AGIPD(
+ memory_cells=max_cells_db_dark,
+ bias_voltage=bias_voltage,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.zeros(
+ (128, 512, max_cells_db, 3)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
+ bpixels = bpixels.astype(np.uint32)
+
+ thresholds, when['thresholds'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
Constants.AGIPD.ThresholdsDark(),
Conditions.Dark.AGIPD(
memory_cells=max_cells_db_dark,
bias_voltage=bias_voltage,
- acquisition_rate=self.acquisition_rate),
+ acquisition_rate=self.acquisition_rate), # noqa
np.ones((128, 512, max_cells_db, 2)),
cal_db_interface,
creation_time=creation_time,
@@ -1494,39 +1508,44 @@ class AgipdCorrections:
return when
# load also non-dark constants from the database
- slopesPC = get_constant_from_db(getattr(dinstance, qm),
- Constants.AGIPD.SlopesPC(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db,
- bias_voltage=bias_voltage,
- acquisition_rate=self.acquisition_rate),
- np.ones((128, 512, max_cells_db, 10)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
+ slopesPC, when['slopesPC'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.SlopesPC(),
+ Conditions.Dark.AGIPD(
+ memory_cells=max_cells_db,
+ bias_voltage=bias_voltage,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.ones(
+ (128, 512, max_cells_db, 10)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
slopesPC = slopesPC.astype(np.float32)
# this will handle some historical data in a slighly different format
- if slopesPC.shape[0] == 10: # constant dimension injected first
+ # constant dimension injected first
+ if slopesPC.shape[0] == 10 or slopesPC.shape[0] == 11:
slopesPC = np.moveaxis(slopesPC, 0, 3)
slopesPC = np.moveaxis(slopesPC, 0, 2)
- slopesFF = np.squeeze(
- get_constant_from_db(getattr(dinstance, qm),
- Constants.AGIPD.SlopesFF(),
- Conditions.Illuminated.AGIPD(max_cells_db,
- bias_voltage,
- photon_energy,
- pixels_x=512,
- pixels_y=128,
- beam_energy=None,
- acquisition_rate=self.acquisition_rate),
- # noqa
- np.ones((128, 512, max_cells_db, 2)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db))
+ slopesFF, when['slopesFF'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.SlopesFF(),
+ Conditions.Illuminated.AGIPD(
+ max_cells_db,
+ bias_voltage,
+ photon_energy,
+ pixels_x=512,
+ pixels_y=128,
+ beam_energy=None,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.ones((128, 512, max_cells_db, 2)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
+ slopesFF = np.squeeze(slopesFF)
+ print(slopesFF.shape, )
# calculate relative gain from the constants
# we default to a value of one, so basically a unity transform
rel_gain = np.ones((128, 512, self.max_cells, 3), np.float32)
@@ -1583,31 +1602,35 @@ class AgipdCorrections:
slopesPC[..., :self.max_cells, 4], xrayOffset]
# add additonal bad pixel information
- bppc = get_constant_from_db(getattr(Detectors.AGIPD1M1, qm),
- Constants.AGIPD.BadPixelsPC(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db,
- bias_voltage=bias_voltage),
- np.zeros((max_cells_db, 128, 512)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
+ bppc, when['bppc'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.BadPixelsPC(),
+ Conditions.Dark.AGIPD(
+ memory_cells=max_cells_db,
+ bias_voltage=bias_voltage,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.zeros((max_cells_db, 128, 512)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
bppc = np.moveaxis(bppc.astype(np.uint32), 0, 2)
bpixels |= bppc[..., :bpixels.shape[2], None]
- bpff = get_constant_from_db(getattr(Detectors.AGIPD1M1, qm),
- Constants.AGIPD.BadPixelsFF(),
- Conditions.Illuminated.AGIPD(max_cells_db,
- bias_voltage,
- photon_energy,
- pixels_x=512,
- pixels_y=128,
- beam_energy=None),
- # noqa
- np.zeros((128, 512, max_cells_db)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
+ bpff, when['bpff'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.BadPixelsFF(),
+ Conditions.Illuminated.AGIPD(
+ max_cells_db,
+ bias_voltage,
+ photon_energy,
+ pixels_x=512,
+ pixels_y=128,
+ beam_energy=None,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.zeros((128, 512, max_cells_db)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
bpixels |= bpff.astype(np.uint32)[..., :bpixels.shape[2], None]
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 23e304ff048c22a7ce1bab632a8dc38a0ebec35f..af85abf7dfe201047a54d2d06a46d790dea57564 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -22,9 +22,9 @@
},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/MID/201931/p900090/raw/\" # the folder to read data from, required\n",
- "run = 5 # runs to process, required\n",
- "out_folder = \"/gpfs/exfel/exp/MID/201931/p900090/proc/\" # the folder to output to, required\n",
+ "in_folder = \"/gpfs/exfel/exp/SPB/201901/p002545/raw\" # the folder to read data from, required\n",
+ "run = 9 # runs to process, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/AGIPD_Corr12\" # the folder to output to, required\n",
"calfile = \"/gpfs/exfel/data/scratch/haufs/agipd_on_demand/agipd_store_mid.h5\" # path to calibration file. Leave empty if all data should come from DB\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
@@ -339,7 +339,8 @@
"ExecuteTime": {
"end_time": "2019-02-21T11:30:16.057429Z",
"start_time": "2019-02-21T11:30:10.082114Z"
- }
+ },
+ "scrolled": false
},
"outputs": [],
"source": [
@@ -404,6 +405,7 @@
" dig_signal_edges = None\n",
" gain_stats = 0\n",
" when = None\n",
+ " err = None\n",
" \n",
" try:\n",
" start = datetime.now()\n",
@@ -414,7 +416,10 @@
" \n",
" if max_cells == 0:\n",
" max_cells = get_num_cells(filename, loc, channel)\n",
- " cells = np.arange(max_cells)\n",
+ " if max_cells is None:\n",
+ " raise ValueError(\"No raw images found for {}\".format(qm))\n",
+ " else:\n",
+ " cells = np.arange(max_cells)\n",
" \n",
" if acq_rate == 0.:\n",
" acq_rate = get_acq_rate(filename, loc, channel)\n",
@@ -464,11 +469,9 @@
" print(\"Initialized constants\")\n",
" \n",
" for irange in agipd_corr.get_iteration_range():\n",
- " \n",
" agipd_corr.correct_agipd(irange)\n",
" print(\"Iterated\")\n",
- " \n",
- " \n",
+ "\n",
" print(\"All interations finished\")\n",
" hists, edges = agipd_corr.get_histograms()\n",
" hists_signal_low, hists_signal_high, hists_gain_vs_signal, hists_dig_gain_vs_signal= hists\n",
@@ -480,6 +483,7 @@
" \n",
" except Exception as e:\n",
" print(e)\n",
+ " err = e\n",
" success = False\n",
" reason = \"Error\"\n",
" \n",
@@ -492,7 +496,7 @@
" #influx = create_influx_entry(run, proposal, qm, sequence, filesize, CHUNK_SIZE, total_sequences, success, duration, reason)\n",
" #client.write_points([influx])\n",
" return (hists_signal_low, hists_signal_high, hists_gain_vs_signal, hists_dig_gain_vs_signal,\n",
- " low_edges, high_edges, signal_edges, dig_signal_edges, gain_stats, max_cells, when)\n",
+ " low_edges, high_edges, signal_edges, dig_signal_edges, gain_stats, max_cells, when, qm, err)\n",
" \n",
"done = False\n",
"first_files = []\n",
@@ -515,6 +519,7 @@
"fileparms = calfile\n",
"all_cells = []\n",
"whens = []\n",
+ "errors = []\n",
"while not done:\n",
" \n",
" dones = []\n",
@@ -550,9 +555,10 @@
"\n",
" for rr in r:\n",
" if rr is not None:\n",
- " hl, hh, hg, hdg, low_edges, high_edges, signal_edges, dig_signal_edges, gs, cells, when = rr\n",
+ " hl, hh, hg, hdg, low_edges, high_edges, signal_edges, dig_signal_edges, gs, cells, when, qm, err = rr\n",
" all_cells.append(cells)\n",
- " whens.append(when)\n",
+ " whens.append((qm, when))\n",
+ " errors.append(err)\n",
" if hl is not None: # any one being None will also make the others None\n",
" hists_signal_low += hl.astype(np.float64)\n",
" hists_signal_high += hh.astype(np.float64)\n",
@@ -560,10 +566,7 @@
" hists_dig_gain_vs_signal += hdg.astype(np.float64)\n",
" gain_stats += gs\n",
" \n",
- " done = all(dones)\n",
- "all_cells = np.array(all_cells)\n",
- "mem_cells = np.min(all_cells[all_cells != 0])\n",
- "max_cells = mem_cells\n"
+ " done = all(dones)"
]
},
{
@@ -572,11 +575,38 @@
"metadata": {},
"outputs": [],
"source": [
+ "print(\"Constants were injected on: \")\n",
+ "to_store = []\n",
+ "for i, (qm, when) in enumerate(whens):\n",
+ " print(qm)\n",
+ " line = [qm]\n",
+ " # If correction is crashed\n",
+ " if errors[i] is not None:\n",
+ " print(\"Error: {}\".format(errors[i]) )\n",
+ " else:\n",
+ " for key, item in when.items():\n",
+ " if hasattr(item, 'strftime'):\n",
+ " item = item.strftime('%y-%m-%d %H:%M')\n",
+ " # If constant retrieval is crashed\n",
+ " else:\n",
+ " item = 'None'\n",
+ " when[key] = item\n",
+ " print('{:.<12s}'.format(key), item)\n",
+ " \n",
+ " # Store few time stamps if exists\n",
+ " # Add NA to keep array structure\n",
+ " for key in ['offset', 'slopesPC', 'slopesFF']:\n",
+ " if when and key in when:\n",
+ " line.append(when[key])\n",
+ " else:\n",
+ " if errors[i] is not None:\n",
+ " line.append('Err')\n",
+ " else:\n",
+ " line.append('NA')\n",
+ " to_store.append(line)\n",
"\n",
- "print(\"Offset where injected on: \")\n",
- "for i, when in enumerate(whens):\n",
- " qm = \"Q{}M{}\".format((i%16)//4 + 1, i%4 +1)\n",
- " print(\"{}: {}\".format(qm, when))"
+ "np.savetxt(\"{}/tmp_const_beginat_S{:05d}.csv\".format(out_folder, sequences[0]), \n",
+ " np.array(to_store).astype(str), fmt='%s', delimiter = ',')"
]
},
{
@@ -781,7 +811,7 @@
" gains.append(np.zeros((last_idx-first_idx, 512, 128)))\n",
" mask.append(np.zeros((last_idx-first_idx, 512, 128)))\n",
" raw.append(np.zeros((last_idx-first_idx, 512, 128)))\n",
- "\n"
+ " "
]
},
{