diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 63a57ef73e2032982710b32b81587af876413122..b96d1b285ddc4a816153c4b4b0eae6657a419af4 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -22,41 +22,49 @@
},
"outputs": [],
"source": [
+ "cluster_profile = \"noDB\"\n",
"in_folder = \"/gpfs/exfel/exp/MID/201931/p900107/raw\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_Corr\" # the folder to output to, required\n",
+ "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
+ "modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
"run = 11 # runs to process, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/karnem/test/AGIPD_Corr\" # the folder to output to, required\n",
+ "\n",
+ "karabo_id = \"MID_DET_AGIPD1M-1\" # karabo karabo_id\n",
+ "karabo_da = [-1] # data aggregators\n",
+ "receiver_id = \"{}CH0\" # inset for receiver devices\n",
+ "path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
+ "h5path = 'INSTRUMENT/{}/DET/{}:xtdf/' # path in the HDF5 file to images\n",
+ "h5path_idx = 'INDEX/{}/DET/{}:xtdf/' # path in the HDF5 file to images\n",
+ "h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP_TEST' # path to control information\n",
+ "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
+ "karabo_da_control = 'DA02' # karabo DA for control infromation\n",
+ "\n",
+ "use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
+ "cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
+ "cal_db_timeout = 30000 # in milli seconds\n",
+ "creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
+ "\n",
"calfile = \"\" # path to calibration file. Leave empty if all data should come from DB\n",
- "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
+ "nodb = False # if set only file-based constants will be used\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
+ "bias_voltage = 300\n",
+ "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
+ "gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
+ "photon_energy = 9.2 # photon energy in keV\n",
"interlaced = False # whether data is in interlaced layout\n",
"overwrite = True # set to True if existing data should be overwritten\n",
- "cluster_profile = \"noDB\"\n",
"max_pulses = [0, 500, 1] # range list [st, end, step] of maximum pulse indices. 3 allowed maximum list input elements. \n",
"local_input = False\n",
- "bias_voltage = 300\n",
- "cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
- "use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
"sequences_per_node = 2 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel\n",
- "photon_energy = 9.2 # photon energy in keV\n",
"index_v = 2 # version of RAW index type\n",
- "nodb = False # if set only file-based constants will be used\n",
"blc_noise_threshold = 5000 # above this mean signal intensity now baseline correction via noise is attempted\n",
- "corr_asic_diag = False # if set, diagonal drop offs on ASICs are correted \n",
"melt_snow = \"\" # if set to \"none\" snowy pixels are identified and resolved to NaN, if set to \"interpolate\", the value is interpolated from neighbouring pixels\n",
- "cal_db_timeout = 30000 # in milli seconds\n",
"max_cells_db_dark = 0 # set to a value different than 0 to use this value for dark data DB queries\n",
"max_cells_db = 0 # set to a value different than 0 to use this value for DB queries\n",
"chunk_size_idim = 1 # chunking size of imaging dimension, adjust if user software is sensitive to this.\n",
- "creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
- "instrument = \"MID\" # the instrument the detector is installed at, required\n",
"force_hg_if_below = 1000 # set to a value other than 0 to force a pixel into high gain if it's high gain offset subtracted value is below this threshold\n",
"force_mg_if_below = 1000 # set to a value other than 0 to force a pixel into medium gain if it's medium gain offset subtracted value is below this threshold\n",
"mask_noisy_adc = 0.25 # set to a value other than 0 and below 1 to mask entire ADC if fraction of noisy pixels is above\n",
- "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
- "gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
- "\n",
- "h5path_ctrl = '/CONTROL/SPB_IRU_AGIPD1M1/MDL/FPGA_COMP_TEST' # path to control information\n",
- "karabo_da_control = 'DA02' # karabo DA for control infromation\n",
"\n",
"# Correction Booleans\n",
"only_offset = False # Apply only Offset correction. if False, Offset is applied by Default. if True, Offset is only applied.\n",
@@ -70,10 +78,11 @@
"dont_zero_nans = False # do not zero NaN values in corrected data\n",
"dont_zero_orange = False # do not zero very negative and very large values\n",
"blc_set_min = False # Shift to 0 negative medium gain pixels after offset corr\n",
+ "corr_asic_diag = False # if set, diagonal drop offs on ASICs are correted \n",
"\n",
- "def balance_sequences(in_folder, run, sequences, sequences_per_node):\n",
+ "def balance_sequences(in_folder, run, sequences, sequences_per_node, karabo_da):\n",
" from xfel_calibrate.calibrate import balance_sequences as bs\n",
- " return bs(in_folder, run, sequences, sequences_per_node, \"AGIPD*\")"
+ " return bs(in_folder, run, sequences, sequences_per_node, karabo_da)\n"
]
},
{
@@ -130,12 +139,12 @@
"matplotlib.use(\"agg\")\n",
"import matplotlib.pyplot as plt\n",
"from ipyparallel import Client\n",
- "print(\"Connecting to profile {}\".format(cluster_profile))\n",
+ "print(f\"Connecting to profile {cluster_profile}\")\n",
"view = Client(profile=cluster_profile)[:]\n",
"view.use_dill()\n",
"\n",
"from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "from cal_tools.tools import (gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name,\n",
+ "from cal_tools.tools import (map_modules_from_folder, parse_runs, run_prop_seq_from_path, get_notebook_name,\n",
" get_dir_creation_date, get_constant_from_db)\n",
"from cal_tools.agipdlib import get_gain_setting\n",
"from dateutil import parser\n",
@@ -152,18 +161,13 @@
" offset = parser.parse(creation_date_offset)\n",
" delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
" creation_time += delta\n",
- " print(\"Using {} as creation time\".format(creation_time))\n",
- "\n",
- "in_folder = \"{}/r{:04d}\".format(in_folder, run)\n",
+ " print(f\"Using {creation_time} as creation time\")\n",
"\n",
- "print(\"Working in IL Mode: {}. Actual cells in use are: {}\".format(il_mode, max_cells))\n",
+ "print(f\"Working in IL Mode: {il_mode}. Actual cells in use are: {max_cells}\")\n",
"\n",
"if sequences[0] == -1:\n",
" sequences = None\n",
- " \n",
- "QUADRANTS = 4\n",
- "MODULES_PER_QUAD = 4\n",
- "DET_FILE_INSET = \"AGIPD\"\n",
+ "\n",
"CHUNK_SIZE = 250\n",
"MAX_PAR = 32\n",
"\n",
@@ -184,16 +188,20 @@
"\n",
"special_opts = blc_noise_threshold, blc_hmatch, melt_snow\n",
"\n",
- "loc = None\n",
+ "instrument = karabo_id.split(\"_\")[0]\n",
"if instrument == \"SPB\":\n",
- " loc = \"SPB_DET_AGIPD1M-1\"\n",
" dinstance = \"AGIPD1M1\"\n",
"else:\n",
- " loc = \"MID_DET_AGIPD1M-1\"\n",
- " dinstance = \"AGIPD1M2\"\n",
- "print(\"Detector in use is {}\".format(loc))"
+ " dinstance = \"AGIPD1M2\"\n"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -215,7 +223,23 @@
" print(\"Set gain settion to 0\")\n",
" gain_setting = 0\n",
" \n",
- "print(\"Gain setting: {}\".format(gain_setting))"
+ "print(f\"Gain setting: {gain_setting}\")\n",
+ "print(f\"Detector in use is {karabo_id}\")\n",
+ "print(f\"Instrument {instrument}\")\n",
+ "print(f\"Detector instance {dinstance}\")\n",
+ "\n",
+ "if karabo_da[0] == -1:\n",
+ " if modules[0] == -1:\n",
+ " modules = list(range(16))\n",
+ " karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
+ "else:\n",
+ " modules = [int(x[-2:]) for x in karabo_da]\n",
+ "print(\"Process modules: \",\n",
+ " ', '.join([f\"Q{x // 4 + 1}M{x % 4 + 1}\" for x in modules]))\n",
+ "\n",
+ "h5path = h5path.format(karabo_id, receiver_id)\n",
+ "h5path_idx = h5path_idx.format(karabo_id, receiver_id)\n",
+ "h5path_ctrl = h5path_ctrl.format(karabo_id_control)"
]
},
{
@@ -288,50 +312,8 @@
"outputs": [],
"source": [
"# set everything up filewise\n",
- "from queue import Queue\n",
- "from collections import OrderedDict\n",
- "\n",
- "def map_modules_from_files(filelist):\n",
- " module_files = OrderedDict()\n",
- " mod_ids = OrderedDict()\n",
- " total_sequences = 0\n",
- " sequences_qm = {}\n",
- " one_module = None\n",
- " for quadrant in range(0, QUADRANTS):\n",
- " for module in range(0, MODULES_PER_QUAD):\n",
- " name = \"Q{}M{}\".format(quadrant + 1, module + 1)\n",
- " module_files[name] = Queue()\n",
- " num = quadrant * 4 + module\n",
- " mod_ids[name] = num\n",
- " file_infix = \"{}{:02d}\".format(DET_FILE_INSET, num)\n",
- " sequences_qm[name] = 0\n",
- " for file in filelist:\n",
- " if file_infix in file:\n",
- " if not one_module:\n",
- " one_module = file, num\n",
- " module_files[name].put(file)\n",
- " total_sequences += 1\n",
- " sequences_qm[name] += 1\n",
- " \n",
- " return module_files, mod_ids, total_sequences, sequences_qm, one_module\n",
- "\n",
- "dirlist = sorted(os.listdir(in_folder))\n",
- "file_list = []\n",
- "\n",
- "\n",
- "for entry in dirlist:\n",
- " #only h5 file\n",
- " abs_entry = \"{}/{}\".format(in_folder, entry)\n",
- " if os.path.isfile(abs_entry) and os.path.splitext(abs_entry)[1] == \".h5\":\n",
- " \n",
- " if sequences is None:\n",
- " file_list.append(abs_entry)\n",
- " else:\n",
- " for seq in sequences:\n",
- " if \"{:05d}.h5\".format(seq) in abs_entry:\n",
- " file_list.append(os.path.abspath(abs_entry))\n",
- " \n",
- "mapped_files, mod_ids, total_sequences, sequences_qm, one_module = map_modules_from_files(file_list)\n",
+ "mmf = map_modules_from_folder(in_folder, run, path_template, karabo_da, sequences)\n",
+ "mapped_files, mod_ids, total_sequences, sequences_qm, _ = mmf\n",
"MAX_PAR = min(MAX_PAR, total_sequences)"
]
},
@@ -356,7 +338,7 @@
"import copy\n",
"from IPython.display import HTML, display, Markdown, Latex\n",
"import tabulate\n",
- "print(\"Processing a total of {} sequence files in chunks of {}\".format(total_sequences, MAX_PAR))\n",
+ "print(f\"Processing a total of {total_sequences} sequence files in chunks of {MAX_PAR}\")\n",
"table = []\n",
"mfc = copy.copy(mapped_files)\n",
"ti = 0\n",
@@ -372,7 +354,8 @@
" ti += 1\n",
"md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=[\"#\", \"module\", \"# module\", \"file\"]))) \n",
"# restore the queue\n",
- "mapped_files, mod_ids, total_sequences, sequences_qm, one_module = map_modules_from_files(file_list)"
+ "mmf = map_modules_from_folder(in_folder, run, path_template, karabo_da, sequences)\n",
+ "mapped_files, mod_ids, total_sequences, sequences_qm, _ = mmf"
]
},
{
@@ -393,7 +376,7 @@
" bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range,\n",
" bins_dig_gain_vs_signal, max_pulses, dbparms, fileparms, nodb, chunk_size_idim,\n",
" special_opts, il_mode, loc, dinstance, force_hg_if_below, force_mg_if_below,\n",
- " mask_noisy_adc, acq_rate, gain_setting, corr_bools, inp):\n",
+ " mask_noisy_adc, acq_rate, gain_setting, corr_bools, h5path, h5path_idx, inp):\n",
" print(\"foo\")\n",
" import numpy as np\n",
" import copy\n",
@@ -479,6 +462,12 @@
" print(\"Set memory cells to {}\".format(max_cells))\n",
" print(\"Set acquistion rate cells to {} MHz\".format(acq_rate))\n",
"\n",
+ " # AGIPD correction requires path without the leading \"/\"\n",
+ " if h5path[0] == '/':\n",
+ " h5path = h5path[1:]\n",
+ " if h5path_idx[0] == '/':\n",
+ " h5path_idx = h5path_idx[1:]\n",
+ " \n",
"\n",
" infile = h5py.File(filename, \"r\", driver=\"core\")\n",
" outfile = h5py.File(filename_out, \"w\")\n",
@@ -488,8 +477,8 @@
" bins_signal_high_range, bins_dig_gain_vs_signal,\n",
" chunk_size_idim=chunk_size_idim,\n",
" il_mode=il_mode, raw_fmt_version=index_v, \n",
- " h5_data_path=\"INSTRUMENT/{}/DET/{{}}CH0:xtdf/\".format(loc),\n",
- " h5_index_path=\"INDEX/{}/DET/{{}}CH0:xtdf/\".format(loc),\n",
+ " h5_data_path=h5path,\n",
+ " h5_index_path=h5path_idx,\n",
" cal_det_instance=dinstance, force_hg_if_below=force_hg_if_below,\n",
" force_mg_if_below=force_mg_if_below, mask_noisy_adc=mask_noisy_adc,\n",
" acquisition_rate=acq_rate, gain_setting=gain_setting,\n",
@@ -602,8 +591,8 @@
" p = partial(correct_module, max_cells, index_v, CHUNK_SIZE, total_sequences,\n",
" sequences_qm, bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range,\n",
" bins_dig_gain_vs_signal, max_pulses, dbparms, fileparms, nodb, chunk_size_idim,\n",
- " special_opts, il_mode, loc, dinstance, force_hg_if_below, force_mg_if_below,\n",
- " mask_noisy_adc, acq_rate, gain_setting, corr_bools)\n",
+ " special_opts, il_mode, karabo_id, dinstance, force_hg_if_below, force_mg_if_below,\n",
+ " mask_noisy_adc, acq_rate, gain_setting, corr_bools, h5path, h5path_idx)\n",
"\n",
" r = view.map_sync(p, inp)\n",
"\n",
@@ -853,23 +842,20 @@
" try:\n",
"\n",
" rf, cf = ff\n",
- " #print(cf, i)\n",
" if rf is None:\n",
" \n",
" raise Exception(\"File not present\")\n",
- " #print(rf)\n",
" infile = h5py.File(rf, \"r\")\n",
- " #print(\"/INSTRUMENT/{}_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/data\".format(instrument, i))\n",
- " raw.append(np.array(infile[\"/INSTRUMENT/{}_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/data\".format(instrument, i)][first_idx:last_idx,0,...]))\n",
+ " datapath = h5path.format(i)\n",
+ " raw.append(np.array(infile[f\"{datapath}/image/data\"][first_idx:last_idx,0,...]))\n",
" infile.close()\n",
" \n",
" infile = h5py.File(cf, \"r\")\n",
- " #print(\"/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/data\".format(i))\n",
- " corrected.append(np.array(infile[\"/INSTRUMENT/{}_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/data\".format(instrument, i)][first_idx:last_idx,...]))\n",
- " gains.append(np.array(infile[\"/INSTRUMENT/{}_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/gain\".format(instrument, i)][first_idx:last_idx,...]))\n",
- " mask.append(np.array(infile[\"/INSTRUMENT/{}_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/mask\".format(instrument, i)][first_idx:last_idx,...]))\n",
- " pulse_ids.append(np.squeeze(infile[\"/INSTRUMENT/{}_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/pulseId\".format(instrument, i)][first_idx:last_idx,...]))\n",
- " train_ids.append(np.squeeze(infile[\"/INSTRUMENT/{}_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/trainId\".format(instrument, i)][first_idx:last_idx,...]))\n",
+ " corrected.append(np.array(infile[f\"{datapath}/image/data\"][first_idx:last_idx,...]))\n",
+ " gains.append(np.array(infile[f\"{datapath}/image/gain\"][first_idx:last_idx,...]))\n",
+ " mask.append(np.array(infile[f\"{datapath}/image/mask\"][first_idx:last_idx,...]))\n",
+ " pulse_ids.append(np.squeeze(infile[f\"{datapath}/image/pulseId\"][first_idx:last_idx,...]))\n",
+ " train_ids.append(np.squeeze(infile[f\"{datapath}/image/trainId\"][first_idx:last_idx,...]))\n",
" infile.close()\n",
" \n",
" except Exception as e:\n",
@@ -1025,20 +1011,6 @@
"h = ax.hist(combined.flatten(), bins=1000, range=(-50, 1000), log=True)\n"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:49.222484Z",
- "start_time": "2019-02-18T17:29:49.219933Z"
- }
- },
- "outputs": [],
- "source": [
- "#np.save('/gpfs/exfel/data/scratch/haufs/agipd_hist/prop_off_pcor_splits.npy', h)"
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -1196,20 +1168,6 @@
" vmax=32, cmap=\"jet\")\n",
"cb = fig.colorbar(im, ax=ax)"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 5dbd4a4b75a1bfa96018f885d1529e1285d07086..ca345b5b104b1a46c06b9d4a01ccd3f0ccede587 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -28,20 +28,33 @@
"in_folder = \"/gpfs/exfel/d/raw/SPB/202030/p900138/\" # path to input data, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD3\" # path to output to, required\n",
"sequences = [0] # sequence files to evaluate.\n",
- "\n",
- "run_high = 199 # run number in which high gain data was recorded, required\n",
- "run_med = 200 # run number in which medium gain data was recorded, required\n",
- "run_low = 201 # run number in which low gain data was recorded, required\n",
- "\n",
- "mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
+ "modules = [-1] # list of modules to evaluate, RANGE ALLOWED\n",
+ "run_high = 264 # run number in which high gain data was recorded, required\n",
+ "run_med = 265 # run number in which medium gain data was recorded, required\n",
+ "run_low = 266 # run number in which low gain data was recorded, required\n",
+ "\n",
+ "karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
+ "karabo_da = [-1] # data aggregators\n",
+ "receiver_id = \"{}CH0\" # inset for receiver devices\n",
+ "path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
+ "h5path = '/INSTRUMENT/{}/DET/{}:xtdf/image' # path in the HDF5 file to images\n",
+ "h5path_idx = '/INDEX/{}/DET/{}:xtdf/image' # path in the HDF5 file to images\n",
+ "h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP_TEST' # path to control information\n",
+ "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
+ "karabo_da_control = 'DA02' # karabo DA for control infromation\n",
+ "\n",
+ "use_dir_creation_date = True # use dir creation date as data production reference date\n",
+ "cal_db_interface = \"tcp://max-exfl016:8020\" # the database interface to use\n",
+ "cal_db_timeout = 3000000 # timeout on caldb requests\"\n",
"local_output = True # output constants locally\n",
"db_output = False # output constants to database\n",
+ "\n",
+ "mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
"bias_voltage = 300 # detector bias voltage\n",
- "cal_db_interface = \"tcp://max-exfl016:8020\" # the database interface to use\n",
- "cal_db_timeout = 3000000 # timeout on caldb requests\"\n",
+ "gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
+ "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
"interlaced = False # assume interlaced data format, for data prior to Dec. 2017\n",
"rawversion = 2 # RAW file format version\n",
- "dont_use_dir_date = False # don't use the dir creation date for determining the creation time\n",
"\n",
"thresholds_offset_sigma = 3. # thresholds in terms of n sigma noise for offset deduced bad pixels\n",
"thresholds_offset_hard = [4000, 8500] # thresholds in absolute ADU terms for offset deduced bad pixels\n",
@@ -49,14 +62,7 @@
"thresholds_noise_sigma = 5. # thresholds in terms of n sigma noise for offset deduced bad pixels\n",
"thresholds_noise_hard = [4, 20] # thresholds in absolute ADU terms for offset deduced bad pixels\n",
"\n",
- "instrument = \"SPB\"\n",
- "high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. Runtime: ~ 1h\n",
- "modules = [0]#,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] # module to run for\n",
- "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
- "gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
- "\n",
- "h5path_ctrl = '/CONTROL/SPB_IRU_AGIPD1M1/MDL/FPGA_COMP_TEST' # path to control information\n",
- "karabo_da_control = 'AGIPD1MCTRL00' # karabo DA for control infromation"
+ "high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. Runtime: ~ 1h"
]
},
{
@@ -87,7 +93,7 @@
"from IPython.display import display, Markdown, Latex\n",
"%matplotlib inline\n",
"\n",
- "from cal_tools.tools import (gain_map_files, parse_runs, \n",
+ "from cal_tools.tools import (map_gain_stages, parse_runs, \n",
" run_prop_seq_from_path, get_notebook_name, \n",
" get_dir_creation_date, save_const_to_h5,\n",
" get_random_db_interface, get_from_db)\n",
@@ -106,42 +112,36 @@
"\n",
"gains = np.arange(3)\n",
"\n",
- "# no need to change this\n",
- "\n",
- "QUADRANTS = 4\n",
- "MODULES_PER_QUAD = 4\n",
- "DET_FILE_INSET = \"AGIPD\"\n",
- "\n",
"IL_MODE = interlaced\n",
"max_cells = mem_cells\n",
" \n",
"offset_runs = OrderedDict()\n",
- "offset_runs[\"high\"] = parse_runs(run_high)[0]\n",
- "offset_runs[\"med\"] = parse_runs(run_med)[0]\n",
- "offset_runs[\"low\"] = parse_runs(run_low)[0]\n",
+ "offset_runs[\"high\"] = run_high\n",
+ "offset_runs[\"med\"] = run_med\n",
+ "offset_runs[\"low\"] = run_low\n",
"\n",
"creation_time=None\n",
- "if not dont_use_dir_date:\n",
+ "if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run_high)\n",
- "\n",
+ "print(f\"Using {creation_time} as creation time of constant.\")\n",
"\n",
"run, prop, seq = run_prop_seq_from_path(in_folder)\n",
- "logger = InfluxLogger(detector=\"AGIPD\", instrument=instrument, mem_cells=mem_cells,\n",
- " notebook=get_notebook_name(), proposal=prop)\n",
- "\n",
- "print(\"Using {} as creation time of constant.\".format(creation_time))\n",
"\n",
"cal_db_interface = get_random_db_interface(cal_db_interface)\n",
- "print('Calibration database interface: {}'.format(cal_db_interface))\n",
+ "print(f'Calibration database interface: {cal_db_interface}')\n",
"\n",
"loc = None\n",
+ "instrument = karabo_id.split(\"_\")[0]\n",
"if instrument == \"SPB\":\n",
- " loc = \"SPB_DET_AGIPD1M-1\"\n",
" dinstance = \"AGIPD1M1\"\n",
"else:\n",
- " loc = \"MID_DET_AGIPD1M-1\"\n",
" dinstance = \"AGIPD1M2\"\n",
- "print(\"Detector in use is {}\".format(loc)) "
+ "print(f\"Detector in use is {karabo_id}\")\n",
+ "print(f\"Instrument {instrument}\")\n",
+ "print(f\"Detector instance {dinstance}\")\n",
+ "\n",
+ "logger = InfluxLogger(detector=\"AGIPD\", instrument=instrument, mem_cells=mem_cells,\n",
+ " notebook=get_notebook_name(), proposal=prop)"
]
},
{
@@ -179,17 +179,28 @@
},
"outputs": [],
"source": [
+ "if karabo_da[0] == -1:\n",
+ " if modules[0] == -1:\n",
+ " modules = list(range(16))\n",
+ " karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
+ "else:\n",
+ " modules = [int(x[-2:]) for x in karabo_da]\n",
+ "\n",
+ "h5path = h5path.format(karabo_id, receiver_id)\n",
+ "h5path_idx = h5path_idx.format(karabo_id, receiver_id)\n",
+ "h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
+ "\n",
"print(\"Parameters are:\")\n",
- "print(\"Proposal: {}\".format(prop))\n",
- "print(\"Memory cells: {}/{}\".format(mem_cells, max_cells))\n",
+ "print(f\"Proposal: {prop}\")\n",
+ "print(f\"Memory cells: {mem_cells}/{max_cells}\")\n",
"print(\"Runs: {}\".format([ v for v in offset_runs.values()]))\n",
- "print(\"Sequences: {}\".format(sequences))\n",
- "print(\"Interlaced mode: {}\".format(IL_MODE))\n",
- "print(\"Using DB: {}\".format(db_output))\n",
- "print(\"Input: {}\".format(in_folder))\n",
- "print(\"Output: {}\".format(out_folder))\n",
- "print(\"Bias voltage: {}V\".format(bias_voltage))\n",
- "print(\"Gain setting: {}\".format(gain_setting))"
+ "print(f\"Sequences: {sequences}\")\n",
+ "print(f\"Interlaced mode: {IL_MODE}\")\n",
+ "print(f\"Using DB: {db_output}\")\n",
+ "print(f\"Input: {in_folder}\")\n",
+ "print(f\"Output: {out_folder}\")\n",
+ "print(f\"Bias voltage: {bias_voltage}V\")\n",
+ "print(f\"Gain setting: {gain_setting}\")"
]
},
{
@@ -211,13 +222,10 @@
"outputs": [],
"source": [
"# set everything up filewise\n",
- "if not os.path.exists(out_folder):\n",
- " os.makedirs(out_folder)\n",
- "\n",
- "gmf = gain_map_files(in_folder, offset_runs, sequences, DET_FILE_INSET, QUADRANTS, MODULES_PER_QUAD)\n",
+ "os.makedirs(out_folder, exist_ok=True)\n",
+ "gmf = map_gain_stages(in_folder, offset_runs, path_template, karabo_da, sequences)\n",
"gain_mapped_files, total_sequences, total_file_size = gmf\n",
- "\n",
- "print(\"Will process at total of {} sequences: {:0.2f} GB of data.\".format(total_sequences, total_file_size))"
+ "print(f\"Will process a total of {total_sequences} sequences.\")"
]
},
{
@@ -242,7 +250,8 @@
"source": [
"import copy\n",
"from functools import partial\n",
- "def characterize_module(il_mode, cells, bp_thresh, rawversion, loc, acq_rate, inp):\n",
+ "def characterize_module(il_mode, cells, bp_thresh, rawversion, loc, acq_rate,\n",
+ " h5path, h5path_idx, inp):\n",
" import numpy as np\n",
" import copy\n",
" import h5py\n",
@@ -251,11 +260,10 @@
" \n",
" filename, filename_out, channel = inp\n",
" \n",
- "\n",
" if cells == 0:\n",
" cells = get_num_cells(filename, loc, channel)\n",
"\n",
- " print(\"Using {} memory cells\".format(cells))\n",
+ " print(f\"Using {cells} memory cells\")\n",
" \n",
" if acq_rate == 0.:\n",
" acq_rate = get_acq_rate(filename, loc, channel)\n",
@@ -263,21 +271,25 @@
" thresholds_offset_hard, thresholds_offset_sigma, thresholds_noise_hard, thresholds_noise_sigma = bp_thresh \n",
"\n",
" infile = h5py.File(filename, \"r\", driver=\"core\")\n",
+ " \n",
+ " h5path = h5path.format(channel)\n",
+ " h5path_idx = h5path_idx.format(channel)\n",
+ " \n",
" if rawversion == 2:\n",
- " count = np.squeeze(infile[\"/INDEX/{}/DET/{}CH0:xtdf/image/count\".format(loc, channel)])\n",
- " first = np.squeeze(infile[\"/INDEX/{}/DET/{}CH0:xtdf/image/first\".format(loc, channel)])\n",
+ " count = np.squeeze(infile[f\"{h5path_idx}/count\"])\n",
+ " first = np.squeeze(infile[f\"{h5path_idx}/first\"])\n",
" last_index = int(first[count != 0][-1]+count[count != 0][-1])\n",
" first_index = int(first[count != 0][0])\n",
" else:\n",
- " status = np.squeeze(infile[\"/INDEX/{}/DET/{}CH0:xtdf/image/status\".format(loc, channel)])\n",
+ " status = np.squeeze(infile[f\"{h5path_idx}/status\"])\n",
" if np.count_nonzero(status != 0) == 0:\n",
" return\n",
- " last = np.squeeze(infile[\"/INDEX/{}/DET/{}CH0:xtdf/image/last\".format(loc, channel)])\n",
- " first = np.squeeze(infile[\"/INDEX/{}/DET/{}CH0:xtdf/image/first\".format(loc, channel)])\n",
+ " last = np.squeeze(infile[f\"{h5path_idx}/last\"])\n",
+ " first = np.squeeze(infile[f\"{h5path_idx}/first\"])\n",
" last_index = int(last[status != 0][-1]) + 1\n",
" first_index = int(first[status != 0][0])\n",
- " im = np.array(infile[\"/INSTRUMENT/{}/DET/{}CH0:xtdf/image/data\".format(loc, channel)][first_index:last_index,...]) \n",
- " cellIds = np.squeeze(infile[\"/INSTRUMENT/{}/DET/{}CH0:xtdf/image/cellId\".format(loc, channel)][first_index:last_index,...]) \n",
+ " im = np.array(infile[f\"{h5path}/data\"][first_index:last_index,...]) \n",
+ " cellIds = np.squeeze(infile[f\"{h5path}/cellId\"][first_index:last_index,...]) \n",
" \n",
" infile.close()\n",
"\n",
@@ -356,9 +368,10 @@
" first = False\n",
" p = partial(characterize_module, IL_MODE, max_cells,\n",
" (thresholds_offset_hard, thresholds_offset_sigma,\n",
- " thresholds_noise_hard, thresholds_noise_sigma), rawversion, loc, acq_rate)\n",
- " results = list(map(p, inp))\n",
- " #results = view.map_sync(p, inp)\n",
+ " thresholds_noise_hard, thresholds_noise_sigma), \n",
+ " rawversion, karabo_id, acq_rate, h5path, h5path_idx)\n",
+ " #results = list(map(p, inp))\n",
+ " results = view.map_sync(p, inp)\n",
" for ii, r in enumerate(results):\n",
" i = modules[ii]\n",
" offset, noise, gain, bp, thiscell, thisacq = r\n",
@@ -383,9 +396,9 @@
" filesize=total_file_size)\n",
"logger.send()\n",
"max_cells = np.max(all_cells)\n",
- "print(\"Using {} memory cells\".format(max_cells))\n",
+ "print(f\"Using {max_cells} memory cells\")\n",
"acq_rate = np.max(all_acq_rate)\n",
- "print(\"Using {} MHz acquisition rate\".format(acq_rate))"
+ "print(f\"Using {acq_rate} MHz acquisition rate\")"
]
},
{
@@ -437,7 +450,9 @@
" \n",
"if local_output:\n",
" for qm in offset_g.keys():\n",
- " ofile = \"{}/agipd_offset_store_{}_{}.h5\".format(out_folder, \"_\".join(offset_runs.values()), qm)\n",
+ " ofile = \"{}/agipd_offset_store_{}_{}.h5\".format(out_folder,\n",
+ " \"{}-{}-{}\".format(*offset_runs.values()), \n",
+ " qm)\n",
" store_file = h5py.File(ofile, \"w\")\n",
" store_file[\"{}/Offset/0/data\".format(qm)] = offset_g[qm]\n",
" store_file[\"{}/Noise/0/data\".format(qm)] = noise_g[qm]\n",
@@ -513,7 +528,7 @@
"source": [
"for qm in res:\n",
"\n",
- " print(\"Injecting constants with conditions:\\n\".format(const))\n",
+ " print(f\"Injecting constants {const} with conditions:\\n\")\n",
" print(\"1. memory_cells: {}\\n2. bias_voltage: {}\\n\"\n",
" \"3. acquisition_rate: {}\\n4. gain_setting: {}\\n\"\n",
" \"5. creation_time: {}\\n\".format(max_cells, bias_voltage,\n",
@@ -636,7 +651,7 @@
"cell = 3\n",
"gain = 0\n",
"out_folder = None\n",
- "show_overview(res, cell, gain, out_folder=out_folder, infix=\"_\".join(offset_runs.values()))"
+ "show_overview(res, cell, gain, out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()))"
]
},
{
@@ -656,7 +671,7 @@
"source": [
"cell = 3\n",
"gain = 1\n",
- "show_overview(res, cell, gain, out_folder=out_folder, infix=\"_\".join(offset_runs.values()))"
+ "show_overview(res, cell, gain, out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()))"
]
},
{
@@ -676,7 +691,7 @@
"source": [
"cell = 3\n",
"gain = 2\n",
- "show_overview(res, cell, gain, out_folder=out_folder, infix=\"_\".join(offset_runs.values()))"
+ "show_overview(res, cell, gain, out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()))"
]
},
{
@@ -768,9 +783,9 @@
},
"outputs": [],
"source": [
- "create_constant_overview(offset_g, \"Offset (ADU)\", max_cells, 4000, 10000,\n",
- " out_folder=out_folder, infix=\"_\".join(offset_runs.values()),\n",
- " badpixels=[badpix_g, np.nan])"
+ "create_constant_overview(offset_g, \"Offset (ADU)\", max_cells, 4000, 8000,\n",
+ " out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()),\n",
+ " badpixels=[badpix_g, np.nan]))"
]
},
{
@@ -782,7 +797,7 @@
"outputs": [],
"source": [
"create_constant_overview(noise_g, \"Noise (ADU)\", max_cells, 0, 100,\n",
- " out_folder=out_folder, infix=\"_\".join(offset_runs.values()),\n",
+ " out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()),\n",
" badpixels=[badpix_g, np.nan])"
]
},
@@ -803,7 +818,7 @@
" \n",
" \n",
"create_constant_overview(thresholds_g, \"Threshold (ADU)\", max_cells, 4000, 10000, 5,\n",
- " out_folder=out_folder, infix=\"_\".join(offset_runs.values()),\n",
+ " out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()),\n",
" badpixels=[bp_thresh, np.nan],\n",
" gmap=['HG-MG Threshold', 'MG-LG Threshold', 'High gain', 'Medium gain', 'low gain'],\n",
" marker=['d','d','','','']\n",
@@ -820,7 +835,7 @@
"for m, d in badpix_g.items():\n",
" bad_pixel_aggregate_g[m] = d.astype(np.bool).astype(np.float)\n",
"create_constant_overview(bad_pixel_aggregate_g, \"Bad pixel fraction\", max_cells, 0, 0.10, 3,\n",
- " out_folder=out_folder, infix=\"_\".join(offset_runs.values()))"
+ " out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()))"
]
},
{