diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 020d86abb3aaafa911055513f5c795826d4c78cf..0b5ba63060d81833ded5fc64383d08860fb29b73 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -1,7 +1,7 @@
import copy
from enum import Enum
import os
-import gc
+
import h5py
import numpy as np
from scipy.signal import cwt, ricker
@@ -13,8 +13,6 @@ from cal_tools.enums import BadPixels
from cal_tools.tools import get_constant_from_db, get_constant_from_db_and_time
from iCalibrationDB import Constants, Conditions, Detectors
-import logging
-
def get_num_cells(fname, loc, module):
with h5py.File(fname, "r") as f:
@@ -116,7 +114,7 @@ class AgipdCorrections:
cal_det_instance="AGIPD1M1", karabo_data_mode=False,
force_hg_if_below=None, force_mg_if_below=None,
mask_noisy_adc=False, acquisition_rate=None, gain_setting=None,
- corr_bools=None, logfile=None):
+ corr_bools=None):
"""
Initialize an AgipdCorrections Class
@@ -194,28 +192,6 @@ class AgipdCorrections:
self.frac_high_med = 0
self.md_additional_offset = 0
self.xray_cor = 0
- self.logfile = logfile
-
- self.logger = logging.getLogger('server_logger2')
- self.logger.setLevel(logging.DEBUG)
- # create file handler which logs even debug messages
- fh = logging.FileHandler(logfile)
- fh.setLevel(logging.DEBUG)
- # create console handler with a higher log level
- ch = logging.StreamHandler()
- ch.setLevel(logging.ERROR)
- # create formatter and add it to the handlers
- formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
- ch.setFormatter(formatter)
- fh.setFormatter(formatter)
- # add the handlers to logger
- self.logger.addHandler(ch)
- self.logger.addHandler(fh)
- fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
- logging.basicConfig(filename=logfile,
- level=getattr(logging, "DEBUG"),
- format=fmt)
-
# check if given corr_bools are correct
tot_corr_bools = ['only_offset', 'adjust_mg_baseline', 'rel_gain',
@@ -932,7 +908,6 @@ class AgipdCorrections:
"""
if not self.initialized:
raise RuntimeError("Must call initialize() first!")
- self.logger.debug("check init \n")
if not self.karabo_data_mode:
agipd_base = self.agipd_base
cidx = self.cidx
@@ -953,7 +928,6 @@ class AgipdCorrections:
# this far end of the image index range we are working on
nidx = int(cidx + irange.size)
else:
- self.logger.debug("Not karabo mode \n")
cidx = 1 # do not produce any histograms
im = irange['image.data']
trainId = np.squeeze(irange['image.trainId'])
@@ -962,15 +936,12 @@ class AgipdCorrections:
cellid = np.squeeze(irange['image.cellId'])
length = np.squeeze(irange['image.length'])
- self.logger.debug("splitting image \n")
# split off image and gain into two separate arrays
im, ga = self.gsfun(im)
- self.logger.debug("float image \n")
# we will work on float data from now on
im = im.astype(np.float32)
- self.logger.debug("copy raw \n")
# some correction require us to maintain a copy of the raw data
raw = copy.copy(im)
@@ -983,7 +954,6 @@ class AgipdCorrections:
length = length[0::2]
# first evaluate the gain into 0, 1, 2 --> high, medium, low
- self.logger.debug("evaluating gain \n")
gain = np.zeros(ga.shape, np.uint8)
gain[...] = 0
t0 = self.thresholds[..., 0]
@@ -994,12 +964,10 @@ class AgipdCorrections:
gain[(ga > t1[cellid, ...]) & (
t1[cellid, ...] > 1.05 * t0[cellid, ...])] = 2
- self.logger.debug("check force_mg_if_below \n")
# force into high or medium gain if requested
if self.force_mg_if_below is not None and self.force_mg_if_below > 0:
gain[(gain == 2) & ((im - self.offset[
cellid, ..., 1]) < self.force_mg_if_below)] = 1
- self.logger.debug("check force_hg_if_below \n")
if self.force_hg_if_below is not None and self.force_hg_if_below > 0:
gain[(gain > 0) & ((im - self.offset[
cellid, ..., 0]) < self.force_hg_if_below)] = 0
@@ -1010,7 +978,6 @@ class AgipdCorrections:
# check if any data has zero standard deviation, and mark this in
# the bad pixel maks
# this can be done on otherwise not corrected data.
- self.logger.debug("check if any data has zero standard deviation \n")
if self.sig_zero_mask is None:
self.sig_zero_mask = np.zeros(
(self.max_cells, im.shape[1], im.shape[2]), np.uint32)
@@ -1018,7 +985,6 @@ class AgipdCorrections:
std = np.nanstd(im[cellid == c, ...], axis=0)
self.sig_zero_mask[
c, std == 0] = BadPixels.DATA_STD_IS_ZERO.value
- self.logger.debug("check the first chunk \n")
# for feedback we produced histograms for the first chunk
if cidx == 0:
H, xe, ye = np.histogram2d(im.flatten(), ga.flatten(),
@@ -1032,31 +998,25 @@ class AgipdCorrections:
range=[[4000, 8192], [0, 4]])
self.hists_dig_gain_vs_signal += H
self.dig_signal_edges = (xe, ye)
- self.logger.debug("check correct constants depending \n")
# now get the correct constants depending on cell id
offsetb = self.offset[cellid, ...]
tmask = self.mask[cellid, ...]
- self.logger.debug("choose constants according to gain setting \n")
# choose constants according to gain setting
off = np.choose(gain,
(offsetb[..., 0], offsetb[..., 1], offsetb[..., 2]))
msk = np.choose(gain, (tmask[..., 0], tmask[..., 1], tmask[..., 2]))
- self.logger.debug("get the correct rel_gain depending on cell-id \n")
# same for relative gain and then bad pixel mask
if hasattr(self, "rel_gain"):
# get the correct rel_gain depending on cell-id
rc = self.rel_gain[cellid, ...]
rel_cor = np.choose(gain, (rc[..., 0], rc[..., 1], rc[..., 2]))
- self.logger.debug("scale raw gain for use in the identifying snowy pixels \n")
# scale raw gain for use in the identifying snowy pixels
rgain = None
if self.melt_snow is not False:
rgain = ga / t0[cellid, ...]
- self.logger.debug("subtract offset \n")
# subtract offset
im -= off
- self.logger.debug("evaluate any baseline shifts \n")
# before doing relative gain correction we need to evaluate any
# baseline shifts
# as they are effectively and additional offset in the data
@@ -1108,27 +1068,22 @@ class AgipdCorrections:
# now we can correct for relative gain if requested
if self.corr_bools.get("rel_gain") and hasattr(self, "rel_gain"):
- self.logger.debug("correct for relative gain \n")
im *= rel_cor
if self.corr_bools.get("adjust_mg_baseline"):
- self.logger.debug("correct adjust_mg_baseline \n")
mgbc = self.md_additional_offset[cellid, ...]
im[gain == 1] += mgbc[gain == 1]
# Set negative values for medium gain to 0
if self.corr_bools.get('blc_set_min'):
- self.logger.debug("correct blc_set_min \n")
im[(im < 0) & (gain == 1)] = 0
# Do xray correction if requested
if self.corr_bools.get("xray_corr"):
- self.logger.debug("correct xray_corr \n")
im *= self.xray_cor
# try to identify snowy pixels at this point
if self.melt_snow is not False:
- self.logger.debug("correct melt_snow\n")
ms = self.melt_snow
im, gain, snowmask = self.melt_snowy_pixels(raw,
im,
@@ -1159,7 +1114,6 @@ class AgipdCorrections:
msk |= self.sig_zero_mask[cellid, ...]
if self.melt_snow is not False:
msk |= snowmask
- self.logger.debug("for the first chunk output some statistics\n")
# for the first chunk output some statistics
if cidx == 0:
copim = copy.copy(im)
@@ -1194,7 +1148,6 @@ class AgipdCorrections:
# apply noisy ADC mask if requested
if self.mask_noisy_adc is not None and self.mask_noisy_adc != 0:
- self.logger.debug("mask_noisy_adc\n")
if self.adc_mask is None:
self.adc_mask = self.make_noisy_adc_mask(msk)
msk |= self.adc_mask
@@ -1203,26 +1156,15 @@ class AgipdCorrections:
sd = 1 if not self.il_mode else 2
if not self.karabo_data_mode:
- self.logger.debug("now write out the data\n")
-
self.ddset[cidx // sd:nidx // sd, ...] = im
- self.logger.debug("1\n")
self.gdset[cidx // sd:nidx // sd, ...] = gain
- self.logger.debug("2\n")
self.mdset[cidx // sd:nidx // sd, ...] = msk
- self.logger.debug("3\n")
self.outfile[agipd_base + "image/cellId"][cidx:nidx] = cellid
- self.logger.debug("4\n")
self.outfile[agipd_base + "image/trainId"][cidx:nidx] = trainId
- self.logger.debug("5\n")
self.outfile[agipd_base + "image/pulseId"][cidx:nidx] = pulseId
- self.logger.debug("6\n")
self.outfile[agipd_base + "image/status"][cidx:nidx] = status
- self.logger.debug("7\n")
self.outfile[agipd_base + "image/length"][cidx:nidx] = length
- self.logger.debug("8\n")
self.cidx = nidx
- self.logger.debug("now write out the data-finished\n")
else:
irange['image.data'] = im
irange['image.gain'] = gain
@@ -1234,7 +1176,6 @@ class AgipdCorrections:
irange['image.length'] = length
return irange
-
def get_valid_image_idx(self):
""" Return the indices of valid data
"""
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index cbf04cb74677ed79a575dddea5a4ccf84b5e1715..afcd3daba756e66c480d60c6350631cfcc015290 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -371,7 +371,7 @@
" bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range,\n",
" bins_dig_gain_vs_signal, max_pulses, dbparms, fileparms, nodb, chunk_size_idim,\n",
" special_opts, il_mode, loc, dinstance, force_hg_if_below, force_mg_if_below,\n",
- " mask_noisy_adc, acq_rate, gain_setting, corr_bools, h5path, h5path_idx, const_yaml, out_folder, inp):\n",
+ " mask_noisy_adc, acq_rate, gain_setting, corr_bools, h5path, h5path_idx, const_yaml, inp):\n",
" print(\"foo\")\n",
" import numpy as np\n",
" import copy\n",
@@ -381,14 +381,12 @@
" from datetime import datetime\n",
" import re\n",
" import os\n",
- " import gc\n",
" # from influxdb import InfluxDBClient\n",
" import subprocess\n",
" from iCalibrationDB import Constants, Conditions, Detectors\n",
" from cal_tools.enums import BadPixels\n",
" from cal_tools.agipdlib import AgipdCorrections, SnowResolution\n",
" from cal_tools.agipdlib import get_num_cells, get_acq_rate\n",
- " import logging \n",
"\n",
" #client = InfluxDBClient('exflqr18318', 8086, 'root', 'root', 'calstats')\n",
" \"\"\"\n",
@@ -432,39 +430,12 @@
" gain_stats = 0\n",
" when = None\n",
" err = None\n",
- " \n",
- " \n",
+ "\n",
" try:\n",
" start = datetime.now()\n",
" success = True\n",
" reason = \"\"\n",
" filename, filename_out, channel, qm = inp\n",
- " \n",
- " base = os.path.basename(filename_out)\n",
- " f = os.path.splitext(base)[0]\n",
- " #Create and configure logger\n",
- " os.makedirs(f\"{out_folder}/logs\", exist_ok=True)\n",
- " logfile=f\"{out_folder}/logs/log_partial_{f}.log\"\n",
- " logfile_2=f\"{out_folder}/logs/log_partial_{f}_2.log\"\n",
- " logger = logging.getLogger('server_logger')\n",
- " logger.setLevel(logging.DEBUG)\n",
- " # create file handler which logs even debug messages\n",
- " fh = logging.FileHandler(logfile)\n",
- " fh.setLevel(logging.DEBUG)\n",
- " # create console handler with a higher log level\n",
- " ch = logging.StreamHandler()\n",
- " ch.setLevel(logging.ERROR)\n",
- " # create formatter and add it to the handlers\n",
- " formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S')\n",
- " ch.setFormatter(formatter)\n",
- " fh.setFormatter(formatter)\n",
- " # add the handlers to logger\n",
- " logger.addHandler(ch)\n",
- " logger.addHandler(fh)\n",
- " fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'\n",
- " logging.basicConfig(filename=logfile,\n",
- " level=getattr(logging, \"DEBUG\"),\n",
- " format=fmt)\n",
"\n",
" if max_cells == 0:\n",
" max_cells = get_num_cells(filename, loc, channel)\n",
@@ -489,7 +460,6 @@
" h5path = h5path[1:]\n",
" if h5path_idx[0] == '/':\n",
" h5path_idx = h5path_idx[1:]\n",
- " \n",
"\n",
" infile = h5py.File(filename, \"r\", driver=\"core\")\n",
" outfile = h5py.File(filename_out, \"w\")\n",
@@ -505,8 +475,7 @@
" cal_det_instance=dinstance, force_hg_if_below=force_hg_if_below,\n",
" force_mg_if_below=force_mg_if_below, mask_noisy_adc=mask_noisy_adc,\n",
" acquisition_rate=acq_rate, gain_setting=gain_setting,\n",
- " corr_bools=corr_bools, logfile=logfile_2)\n",
- " logger.debug(\"init_class \\n\")\n",
+ " corr_bools=corr_bools)\n",
" blc_noise_threshold, blc_hmatch, melt_snow = special_opts\n",
" if not corr_bools[\"only_offset\"]:\n",
" blc_hmatch = False\n",
@@ -517,14 +486,9 @@
" agipd_corr.get_valid_image_idx()\n",
" except IOError:\n",
" return\n",
- " logger.debug(\"getting valid_image \\n\")\n",
"\n",
" device = getattr(getattr(Detectors, dinstance), qm)\n",
"\n",
- " # check if there is a yaml file in out_folder that has the device constants.\n",
- " slag = fileparms != \"\"\n",
- " logger.debug(f\"retrieving constants. yaml: {const_yaml}, not nodb: {not nodb}, fileparms: {slag}, device [{device}] {device.device_name in const_yaml}\\n\") \n",
- "\n",
" if not nodb:\n",
" if const_yaml and device.device_name in const_yaml:\n",
" print(fileparms != \"\")\n",
@@ -538,41 +502,29 @@
" agipd_corr.initialize_from_file(fileparms, qm, with_dark=nodb)\n",
" print(\"Initialized constants\")\n",
"\n",
- " logger.debug(\"retrieved constants \\n\") \n",
- "\n",
" for irange in agipd_corr.get_iteration_range():\n",
" agipd_corr.correct_agipd(irange)\n",
- " # print(\"Iterated\")\n",
- " # logger.debug(f\"{irange}\\n\")\n",
- " # clear parameters\n",
- " #agipd_corr.clear_constants()\n",
- " logger.debug(\"All iterations are finished \\n\")\n",
+ " print(\"Iterated\")\n",
" hists, edges = agipd_corr.get_histograms()\n",
" hists_signal_low, hists_signal_high, hists_gain_vs_signal, hists_dig_gain_vs_signal, hist_pulses = hists\n",
" low_edges, high_edges, signal_edges, dig_signal_edges = edges\n",
" gain_stats = np.array(agipd_corr.gain_stats)\n",
- " logger.debug(\"history_data \\n\")\n",
" finally:\n",
" outfile.close()\n",
" infile.close()\n",
- " logger.debug(\"closed files \\n\")\n",
" print(\"Closed files\")\n",
" except Exception as e:\n",
" err = f\"Error: {e}\\nError traceback: {traceback.format_exc()}\"\n",
" print(err)\n",
- " logger.debug(f\"{err} \\n\")\n",
" success = False\n",
" reason = \"Error\"\n",
" \n",
" finally:\n",
- " logger.debug(\"finally\\n\")\n",
" run = re.findall(r'.*r([0-9]{4}).*', filename)[0]\n",
" proposal = re.findall(r'.*p([0-9]{6}).*', filename)[0]\n",
" sequence = re.findall(r'.*S([0-9]{5}).*', filename)[0]\n",
" filesize = os.path.getsize(filename)\n",
" duration = (datetime.now()-start).total_seconds()\n",
- " logger.debug(f\"run:{run}, proposal:{proposal}, qm:{qm}, sequence:{sequence}, filesize:{filesize}, CHUNK_SIZE:{CHUNK_SIZE}, total_sequences:{total_sequences}, success:{success}, duration:{duration}, reason:{reason}\\n\")\n",
- " logger.debug(\"finished \\n\")\n",
" #influx = create_influx_entry(run, proposal, qm, sequence, filesize, CHUNK_SIZE, total_sequences, success, duration, reason)\n",
" #client.write_points([influx])\n",
" return (hists_signal_low, hists_signal_high, hists_gain_vs_signal, hists_dig_gain_vs_signal, hist_pulses,\n",
@@ -648,8 +600,7 @@
" sequences_qm, bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range,\n",
" bins_dig_gain_vs_signal, max_pulses, dbparms, fileparms, nodb, chunk_size_idim,\n",
" special_opts, il_mode, karabo_id, dinstance, force_hg_if_below, force_mg_if_below,\n",
- " mask_noisy_adc, acq_rate, gain_setting, corr_bools, h5path, h5path_idx, const_yaml, out_folder)\n",
- " print(\"MAX_PAR\", MAX_PAR, \"left\", left, \"\\n\")\n",
+ " mask_noisy_adc, acq_rate, gain_setting, corr_bools, h5path, h5path_idx, const_yaml)\n",
" r = view.map_sync(p, inp)\n",
" #r = list(map(p, inp))\n",
"\n",
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 655182d2d13574fb14c4da0b1fe17889d4ef0b33..8f3df25d24065fb556f678d4e27dc4618fa06532 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -338,7 +338,7 @@
"\n",
"# A dict to connect virtual device\n",
"# to actual device name.\n",
- "for i in range(len(modules)):\n",
+ "for i in modules:\n",
" qm = f\"Q{i//4+1}M{i%4+1}\"\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" device = getattr(getattr(Detectors, dinstance), qm)\n",
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index c745c0f4c62598978547b3eefdb5f6525709dc0d..57984dbf88dcdb7ac6f6ac49ad4715654914794f 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -601,7 +601,7 @@ def get_launcher_command(args, temp_path, dep_jids):
if args.get('slurm_priority'):
launcher_slurm += " --nice={}".format(args.get('slurm_priority'))
- launcher_slurm += " --mem {}G".format(args.get('slurm_mem', '500'))
+ launcher_slurm += " --mem {}G".format(args.get('slurm_mem', '750'))
if len(dep_jids):
srun_dep = " --dependency=afterok"