diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index 4189dd347f4deb7bdd4c744a2e26e0b4894ceff4..2c485c79b17d0a474e0a85d59c2da8407c678218 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -27,11 +27,9 @@
"karabo_id = \"SPB_IRDA_JF4M\" # karabo prefix of Jungfrau devices\n",
"karabo_da = ['JNGFR01', 'JNGFR02', 'JNGFR03', 'JNGFR04', 'JNGFR05', 'JNGFR06', 'JNGFR07', 'JNGFR08'] # data aggregators\n",
"receiver_template = \"JNGFR{:02d}\" # Detector receiver template for accessing raw data files. e.g. \"JNGFR{:02d}\"\n",
- "path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # template to use for file name\n",
"instrument_source_template = '{}/DET/{}:daqOutput' # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'\n",
- "ctrl_source_template = '{}/DET/CONTROL' # template for control source name (filled with karabo_id).\n",
+ "ctrl_source_template = '{}/DET/CONTROL' # template for control source name (filled with karabo_id_control)\n",
"karabo_id_control = \"\" # if control is on a different ID, set to empty string if it is the same a karabo-id\n",
- "karabo_da_control = \"JNGFRCTRL00\" # file inset for control data\n",
"\n",
"# Parameters for calibration database.\n",
"use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
@@ -44,11 +42,12 @@
"plt_images = 100 # Number of images to plot after applying selected corrections.\n",
"limit_images = 0 # ONLY FOR TESTING. process only first N images, Use 0 to process all.\n",
"\n",
- "\n",
"# Parameters for retrieving calibration constants\n",
"manual_slow_data = False # if true, use manually entered bias_voltage and integration_time values\n",
"integration_time = 4.96 # integration time in us, will be overwritten by value in file\n",
"gain_setting = 0 # 0 for dynamic gain, 1 for dynamic HG0, will be overwritten by value in file\n",
+ "gain_mode = 0 # 0 for runs with dynamic gain setting, 1 for fixgain. It will be overwritten by value in file, if manual_slow_data is set to True.
+",
"mem_cells = 0 # leave memory cells equal 0, as it is saved in control information starting 2019.\n",
"bias_voltage = 180 # will be overwritten by value in file\n",
"\n",
@@ -196,12 +195,15 @@
"if not manual_slow_data:\n",
" integration_time = ctrl_data.get_integration_time()\n",
" bias_voltage = ctrl_data.get_bias_voltage()\n",
- " gain_str, gain_setting = ctrl_data.get_gain_setting()\n",
+ " gain_setting = ctrl_data.get_gain_setting()\n",
+ " gain_mode = ctrl_data.get_gain_mode()\n",
"\n",
"print(f\"Integration time is {integration_time} us\")\n",
- "print(f\"Gain setting is {gain_setting} ({gain_str})\")\n",
+ "print(f\"Gain setting is {gain_setting} (run settings: \"\n",
+ " f\"{ctrl_data.run_settings.value if ctrl_data.run_settings else ctrl_data.run_settings})\") # noqa\n",
+ "print(f\"Gain mode is {gain_mode}\")\n",
"print(f\"Bias voltage is {bias_voltage} V\")\n",
- "print(f\"Number of memory cells is {memory_cells}\")"
+ "print(f\"Number of memory cells are {memory_cells}\")"
]
},
{
@@ -217,12 +219,19 @@
"metadata": {},
"outputs": [],
"source": [
- "condition = Conditions.Dark.jungfrau(\n",
- " memory_cells=memory_cells,\n",
- " bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " gain_setting=gain_setting,\n",
- ")\n",
+ "condition = {\n",
+ " \"memory_cells\": memory_cells,\n",
+ " \"bias_voltage\": bias_voltage,\n",
+ " \"integration_time\": integration_time,\n",
+ " \"gain_setting\": gain_setting,\n",
+ "}\n",
+ "# TODO: Currently there are no gain constants for fixed gain mode.\n",
+ "# This should not be the case in the future.\n",
+ "gain_condition = Conditions.Dark.jungfrau(**condition)\n",
+ "\n",
+ "condition[\"gain_mode\"] = gain_mode\n",
+ "dark_condition = Conditions.Dark.jungfrau(**condition)\n",
+ "\n",
"\n",
"def get_constants_for_module(karabo_da: str):\n",
" \"\"\" Get calibration constants for given module of Jungfrau\n",
@@ -240,25 +249,28 @@
" get_constant_from_db_and_time,\n",
" karabo_id=karabo_id,\n",
" karabo_da=karabo_da,\n",
- " condition=condition,\n",
" cal_db_interface=cal_db_interface,\n",
" creation_time=creation_time,\n",
" timeout=cal_db_timeout,\n",
" print_once=False,\n",
" )\n",
" offset_map, when[\"Offset\"] = retrieval_function(\n",
+ " condition=dark_condition,\n",
" constant=Constants.jungfrau.Offset(),\n",
- " empty_constant=np.zeros((1024, 512, 1, 3))\n",
+ " empty_constant=np.zeros((512, 1024, memory_cells, 3))\n",
" )\n",
" mask, when[\"BadPixelsDark\"] = retrieval_function(\n",
+ " condition=dark_condition,\n",
" constant=Constants.jungfrau.BadPixelsDark(),\n",
- " empty_constant=np.zeros((1024, 512, 1, 3)),\n",
+ " empty_constant=np.zeros((512, 1024, memory_cells, 3), dtype=np.uint32),\n",
" )\n",
" mask_ff, when[\"BadPixelsFF\"] = retrieval_function(\n",
+ " condition=gain_condition,\n",
" constant=Constants.jungfrau.BadPixelsFF(),\n",
" empty_constant=None\n",
" )\n",
" gain_map, when[\"Gain\"] = retrieval_function(\n",
+ " condition=gain_condition,\n",
" constant=Constants.jungfrau.RelativeGain(),\n",
" empty_constant=None\n",
" )\n",
@@ -267,20 +279,22 @@
" if mask_ff is not None:\n",
" mask |= np.moveaxis(mask_ff, 0, 1)\n",
"\n",
- " # move from x,y,cell,gain to cell,x,y,gain\n",
- " offset_map = np.squeeze(offset_map)\n",
- " mask = np.squeeze(mask)\n",
- "\n",
" if memory_cells > 1:\n",
- " offset_map = np.moveaxis(np.moveaxis(offset_map, 0, 2), 0, 2)\n",
- " mask = np.moveaxis(np.moveaxis(mask, 0, 2), 0, 2)\n",
+ " # move from x, y, cell, gain to cell, x, y, gain\n",
+ " offset_map = np.moveaxis(offset_map, [0, 1], [1, 2])\n",
+ " mask = np.moveaxis(mask, [0, 1], [1, 2])\n",
+ " else:\n",
+ " offset_map = np.squeeze(offset_map)\n",
+ " mask = np.squeeze(mask)\n",
"\n",
" if gain_map is not None:\n",
" if memory_cells > 1:\n",
- " gain_map = np.moveaxis(np.moveaxis(gain_map, 0, 2), 0, 1)\n",
+ " gain_map = np.moveaxis(gain_map, [0, 2], [2, 0])\n",
+ " # add extra empty cell constant\n",
+ " b = np.ones(((1,)+gain_map.shape[1:]))\n",
+ " gain_map = np.concatenate((gain_map, b), axis=0)\n",
" else:\n",
- " gain_map = np.squeeze(gain_map)\n",
- " gain_map = np.moveaxis(gain_map, 1, 0)\n",
+ " gain_map = np.moveaxis(np.squeeze(gain_map), 1, 0)\n",
"\n",
" return offset_map, mask, gain_map, karabo_da, when\n",
"\n",
@@ -309,28 +323,27 @@
"source": [
"# Correct a chunk of images for offset and gain\n",
"def correct_train(wid, index, d):\n",
- " d = d.astype(np.float32) # [2, x, y]\n",
+ " d = d.astype(np.float32) # [cells, x, y]\n",
" g = gain[index]\n",
- " m = memcells[index]\n",
"\n",
+ " # Jungfrau gains 0[00], 1[01], 3[11]\n",
" g[g==3] = 2\n",
"\n",
- " if 0 <= index < plt_images:\n",
- " r_data[index, ...] = d[0, ...]\n",
- " if memory_cells == 1:\n",
- " g_data[index, ...] = g\n",
- " else:\n",
- " g_data[index, ...] = g[0, ...]\n",
" # Select memory cells\n",
- " \n",
- " # TODO: This needs to be revisited.\n",
- " # As this result in copying data to a new array on every train,\n",
- " # even when there's the same pattern of memory cells on every train.\n",
" if memory_cells > 1:\n",
- " m[m>16] = 0 # TODO: this is wrong and needs to be updated with burst mode.\n",
- " # For an invalid image a memory cell of 255 is set.\n",
- " # These images doesn't need to be processed.\n",
- " offset_map_cell = offset_map[m, ...]\n",
+ " \"\"\"\n",
+ " Even though it is correct to assume that memory cells pattern \n",
+ " can be the same across all trains (for one correction run\n",
+ " taken with one acquisition), it is preferred to not assume\n",
+ " this to account for exceptions that can happen.\n",
+ " \"\"\"\n",
+ " m = memcells[index].copy()\n",
+ " # 255 is a cell value pointing to no cell image data (image of 0 pixels).\n",
+ " # Corresponding image will be corrected with constant of cell 0. To avoid values of 0.\n",
+ " # This line is depending on not storing the modified memory cells in the corrected data.\n",
+ " m[m==255] = 0\n",
+ "\n",
+ " offset_map_cell = offset_map[m, ...] # [16 + empty cell, x, y]\n",
" mask_cell = mask[m, ...]\n",
" else:\n",
" offset_map_cell = offset_map\n",
@@ -338,6 +351,7 @@
"\n",
" # Offset correction\n",
" offset = np.choose(g, np.moveaxis(offset_map_cell, -1, 0))\n",
+ "\n",
" d -= offset\n",
"\n",
" # Gain correction\n",
@@ -347,7 +361,6 @@
" else:\n",
" gain_map_cell = gain_map\n",
" cal = np.choose(g, np.moveaxis(gain_map_cell, -1, 0))\n",
- "\n",
" d /= cal\n",
"\n",
" msk = np.choose(g, np.moveaxis(mask_cell, -1, 0))\n",
@@ -379,9 +392,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "tags": []
- },
+ "metadata": {},
"outputs": [],
"source": [
"fim_data = {}\n",
@@ -432,14 +443,6 @@
" offset_map, mask, gain_map = constants[local_karabo_da]\n",
"\n",
" # Allocate shared arrays.\n",
- "\n",
- " # Shared arrays for plotting.\n",
- " g_data = context.alloc(\n",
- " shape=(plt_images, dshape[2], dshape[3]), dtype=np.uint8)\n",
- " r_data = context.alloc(\n",
- " shape=(plt_images, dshape[2], dshape[3]), dtype=np.float32)\n",
- "\n",
- " # shared arrays for corrected data.\n",
" data_corr = context.alloc(shape=dshape, dtype=np.float32)\n",
" mask_corr = context.alloc(shape=dshape, dtype=np.uint32)\n",
"\n",
@@ -451,6 +454,8 @@
" gain = seq_dc[instrument_src_kda, \"data.gain\"].ndarray()\n",
" memcells = seq_dc[instrument_src_kda, \"data.memoryCell\"].ndarray()\n",
"\n",
+ " rim_data[local_karabo_da] = data[:plt_images, 0, ...].copy()\n",
+ "\n",
" context.map(correct_train, data)\n",
" step_timer.done_step(f'Correction time.')\n",
"\n",
@@ -482,12 +487,13 @@
"\n",
" # Prepare plotting arrays\n",
" step_timer.start()\n",
+ "\n",
" # TODO: Print out which cell is being selected for plotting\n",
" fim_data[local_karabo_da] = data_corr[:plt_images, 0, ...].copy()\n",
" msk_data[local_karabo_da] = mask_corr[:plt_images, 0, ...].copy()\n",
- " gim_data[local_karabo_da] = g_data\n",
- " rim_data[local_karabo_da] = r_data\n",
- " step_timer.done_step(f'Preparing plotting data of {plt_images} images:.')"
+ " gim_data[local_karabo_da] = gain[:plt_images, 0, ...].copy()\n",
+ "\n",
+ " step_timer.done_step(f'Preparing plotting data of {plt_images} images.')"
]
},
{
@@ -552,7 +558,7 @@
" do_2d_plot(\n",
" h, (ex, ey),\n",
" \"Signal (ADU)\",\n",
- " \"Gain Bit Value\",\n",
+ " \"Gain Bit Value (high gain=0[00], medium gain=1[01], low gain=3[11])\",\n",
" f\"Module {mod}\")"
]
},
diff --git a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
index f756d2cb3260ba34ff6cd488940e6a8e04017477..b90a78ed89c02977a660005112e699aac5d4cc47 100644
--- a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
@@ -28,11 +28,8 @@
"karabo_id = 'SPB_IRDA_JF4M' # karabo_id (detector identifier) prefix of Jungfrau detector to process.\n",
"karabo_id_control = '' # if control is on a different ID, set to empty string if it is the same a karabo-id\n",
"receiver_template = 'JNGFR{:02}' # inset for receiver devices\n",
- "receiver_control_id = \"CONTROL\" # inset for control devices\n",
- "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # template to use for file name, double escape sequence number\n",
"instrument_source_template = '{}/DET/{}:daqOutput' # template for instrument source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'\n",
- "ctrl_source_template = '{}/DET/CONTROL' # template for control source name (filled with karabo_id).\n",
- "karabo_da_control = 'JNGFRCTRL00' # file inset for control data\n",
+ "ctrl_source_template = '{}/DET/CONTROL' # template for control source name (filled with karabo_id_control)\n",
"\n",
"# Parameters for calibration database and storing constants.\n",
"use_dir_creation_date = True # use dir creation date\n",
@@ -45,19 +42,21 @@
"badpixel_threshold_sigma = 5. # bad pixels defined by values outside n times this std from median\n",
"offset_abs_threshold_low = [1000, 10000, 10000] # absolute bad pixel threshold in terms of offset, lower values\n",
"offset_abs_threshold_high = [8000, 15000, 15000] # absolute bad pixel threshold in terms of offset, upper values\n",
- "max_trains = 500 # maximum trains to process darks from.\n",
+ "max_trains = 0 # Maximum trains to process darks. Set to 0 to process all available train images.\n",
"min_trains = 1 # Minimum number of trains that should be available to process dark constants. Default 1.\n",
"manual_slow_data = False # if true, use manually entered bias_voltage and integration_time values\n",
"time_limits = 0.025 # to find calibration constants later on, the integration time is allowed to vary by 0.5 us\n",
"\n",
"# Parameters to be used for injecting dark calibration constants.\n",
"integration_time = 1000 # integration time in us, will be overwritten by value in file\n",
- "gain_setting = 0 # 0 for dynamic, forceswitchg1, forceswitchg2, 1 for dynamichg0, fixedgain1, fixgain2. Will be overwritten by value in file\n",
+ "gain_setting = 0 # 0 for dynamic, forceswitchg1, forceswitchg2, 1 for dynamichg0, fixgain1, fixgain2. Will be overwritten by value in file\n",
+ "gain_mode = 0 # 1 if medium and low runs are fixgain1 and fixgain2, otherwise 0. It will be overwritten by value in file, if manual_slow_data\n",
"bias_voltage = 90 # sensor bias voltage in V, will be overwritten by value in file\n",
"memory_cells = 16 # number of memory cells\n",
"\n",
- "# Don't remove. myMDC sends this by default.\n",
- "operation_mode = '' # Detector operation mode, optional\n",
+ "# TODO: this is used for only Warning check at AGIPD dark.\n",
+ "# Need to rethink if it makes sense to use it here as well.\n",
+ "operation_mode = 'ADAPTIVE_GAIN' # Detector operation mode, optional\n",
"\n",
"# TODO: Remove\n",
"db_module = [\"\"] # ID of module in calibration database. TODO: remove from calibration_configurations."
@@ -75,12 +74,14 @@
"import os\n",
"import warnings\n",
"from pathlib import Path\n",
- "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
+ "import multiprocessing\n",
"import numpy as np\n",
+ "import pasha as psh\n",
+ "from IPython.display import Markdown, display\n",
"from extra_data import RunDirectory\n",
"\n",
"matplotlib.use('agg')\n",
@@ -90,7 +91,7 @@
"from XFELDetAna.plotting.histogram import histPlot\n",
"from cal_tools import jungfraulib, step_timing\n",
"from cal_tools.ana_tools import save_dict_to_hdf5\n",
- "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.enums import BadPixels, JungfrauSettings\n",
"from cal_tools.tools import (\n",
" get_dir_creation_date,\n",
" get_pdu_from_db,\n",
@@ -113,6 +114,11 @@
"sensor_size = (1024, 512)\n",
"gains = [0, 1, 2]\n",
"\n",
+ "fixed_settings = [\n",
+ " JungfrauSettings.FIX_GAIN_1.value, JungfrauSettings.FIX_GAIN_2.value] # noqa\n",
+ "dynamic_settings = [\n",
+ " JungfrauSettings.FORCE_SWITCH_HG1.value, JungfrauSettings.FORCE_SWITCH_HG2.value] # noqa\n",
+ "\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run_high)\n",
@@ -122,8 +128,6 @@
"cal_db_interface = get_random_db_interface(cal_db_interface)\n",
"print(f'Calibration database interface: {cal_db_interface}')\n",
"\n",
- "offset_abs_threshold = [offset_abs_threshold_low, offset_abs_threshold_high]\n",
- "\n",
"if karabo_id_control == \"\":\n",
" karabo_id_control = karabo_id"
]
@@ -157,29 +161,28 @@
"source": [
"step_timer.start()\n",
"gain_runs = dict()\n",
- "noise_map = dict()\n",
- "offset_map = dict()\n",
- "gain_str = None\n",
+ "\n",
+ "med_low_settings = []\n",
+ "\n",
"ctrl_src = ctrl_source_template.format(karabo_id_control)\n",
"\n",
"for gain, run_n in enumerate(run_nums):\n",
" run_dc = RunDirectory(f\"{in_folder}/r{run_n:04d}/\")\n",
" gain_runs[run_n] = [gain, run_dc]\n",
- "\n",
+ " ctrl_data = jungfraulib.JungfrauCtrl(run_dc, ctrl_src)\n",
+ " run_settings = ctrl_data.run_settings.value if ctrl_data.run_settings else ctrl_data.run_settings # noqa\n",
" # Read control data for the high gain run only.\n",
" if run_n == run_high:\n",
- " ctrl_data = jungfraulib.JungfrauCtrl(run_dc, ctrl_src)\n",
+ "\n",
" run_mcells, sc_start = ctrl_data.get_memory_cells()\n",
"\n",
" if not manual_slow_data:\n",
" integration_time = ctrl_data.get_integration_time()\n",
" bias_voltage = ctrl_data.get_bias_voltage()\n",
- " gain_str, gain_setting = ctrl_data.get_gain_setting()\n",
- "\n",
- " print(f\"Gain setting is {gain_setting} ({gain_str})\")\n",
- " print(f\"Integration time is {integration_time} us\")\n",
- " print(f\"Bias voltage is {bias_voltage} V\")\n",
- "\n",
+ " gain_setting = ctrl_data.get_gain_setting()\n",
+ " print(f\"Gain setting is {gain_setting} ({run_settings})\")\n",
+ " print(f\"Integration time is {integration_time} us\")\n",
+ " print(f\"Bias voltage is {bias_voltage} V\")\n",
" if run_mcells == 1:\n",
" memory_cells = 1\n",
" print('Dark runs in single cell mode, '\n",
@@ -188,21 +191,76 @@
" memory_cells = 16\n",
" print('Dark runs in burst mode, '\n",
" f'storage cell start: {sc_start:02d}')\n",
- "step_timer.done_step(f'Reading control data.')\n",
- "# Initialize noise_map and offset_map module arrays.\n",
- "for mod in karabo_da:\n",
- " noise_map[mod] = np.zeros(sensor_size+(memory_cells, 3))\n",
- " offset_map[mod] = np.zeros(sensor_size+(memory_cells, 3))"
+ " else:\n",
+ " gain_mode = ctrl_data.get_gain_mode()\n",
+ "\n",
+ " med_low_settings.append(run_settings)\n",
+ "\n",
+ "# A transperent workaround for old raw data with wrong/missing medium and low settings\n",
+ "if med_low_settings == [None, None]:\n",
+ " print(\"WARNING: run.settings is not stored in the data to read. \"\n",
+ " f\"Hence assuming gain_mode = {gain_mode} for adaptive old data.\")\n",
+ "elif med_low_settings == [\"dynamicgain\", \"forceswitchg1\"]:\n",
+ " print(f\"WARNING: run.settings for medium and low gain runs are wrong {med_low_settings}. \"\n",
+ " f\"This is an expected bug for old raw data. Setting gain_mode to {gain_mode}.\")\n",
+ "# Validate that low_med_settings is not a mix of adaptive and fixed settings.\n",
+ "elif not (sorted(med_low_settings) in [fixed_settings, dynamic_settings]): # noqa\n",
+ " raise ValueError(\n",
+ " \"Medium and low run settings are not as expected. \"\n",
+ " f\"Either {dynamic_settings} or {fixed_settings} are expected.\\n\"\n",
+ " f\"Got {sorted(med_low_settings)} for both runs, respectively.\")\n",
+ "\n",
+ "print(f\"Gain mode is {gain_mode} ({med_low_settings})\")\n",
+ "\n",
+ "step_timer.done_step(f'Reading control data.')"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "tags": []
- },
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Use only high gain threshold for all gains in case of fixed_gain.\n",
+ "\n",
+ "if gain_mode: # fixed_gain\n",
+ " offset_abs_threshold = [[offset_abs_threshold_low[0]]*3, [offset_abs_threshold_high[0]]*3]\n",
+ "else:\n",
+ " offset_abs_threshold = [offset_abs_threshold_low, offset_abs_threshold_high]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "context = psh.context.ThreadContext(num_workers=multiprocessing.cpu_count())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
"outputs": [],
"source": [
+ "\"\"\"\n",
+ "All jungfrau runs are taken through one acquisition, except for the forceswitch runs.\n",
+ "While taking non-fixed dark runs, a procedure of multiple acquisitions is used to switch the storage cell indices.\n",
+ "\n",
+ "This is done for medium and low gain dark dynamic runs, only [forceswitchg1, forceswitchg2]:\n",
+ "Switching the cell indices in burst mode is a work around for hardware procedure\n",
+ "deficiency that produces wrong data for dark runs except for the first storage cell.\n",
+ "This is why multiple acquisitions are taken to switch the used storage cells and\n",
+ "acquire data through two cells for each of the 16 cells instead of acquiring darks through all 16 cells.\n",
+ "\"\"\"\n",
+ "\n",
+ "print(f\"Maximum trains to process is set to {max_trains}\")\n",
+ "\n",
+ "noise_map = dict()\n",
+ "offset_map = dict()\n",
+ "bad_pixels_map = dict()\n",
+ "\n",
"for mod in karabo_da:\n",
" step_timer.start()\n",
" instrument_src = instrument_source_template.format(\n",
@@ -210,8 +268,23 @@
"\n",
" print(f\"\\n- Instrument data path for {mod} is {instrument_src}.\")\n",
"\n",
+ " offset_map[mod] = context.alloc(shape=(sensor_size+(memory_cells, 3)), fill=0)\n",
+ " noise_map[mod] = context.alloc(like=offset_map[mod], fill=0)\n",
+ " bad_pixels_map[mod] = context.alloc(like=offset_map[mod], dtype=np.uint32, fill=0)\n",
+ "\n",
" for run_n, [gain, run_dc] in gain_runs.items():\n",
"\n",
+ " def process_cell(worker_id, array_index, cell_number):\n",
+ " cell_slice_idx = acelltable == cell_number\n",
+ " thiscell = images[..., cell_slice_idx]\n",
+ "\n",
+ " offset_map[mod][..., cell_number, gain] = np.mean(thiscell, axis=2)\n",
+ " noise_map[mod][..., cell_number, gain] = np.std(thiscell, axis=2)\n",
+ "\n",
+ " # Check if there are wrong bad gain values.\n",
+ " # Indicate pixels with wrong gain value across all trains for each cell.\n",
+ " bad_pixels_map[mod][\n",
+ " np.average(gain_vals[..., cell_slice_idx], axis=2) != raw_g] |= BadPixels.WRONG_GAIN_VALUE.value\n",
" print(f\"Gain stage {gain}, run {run_n}\")\n",
"\n",
" # load shape of data for memory cells, and detector size (imgs, cells, x, y)\n",
@@ -219,14 +292,14 @@
" # load number of data available, including trains with empty data.\n",
" n_trains = len(run_dc.train_ids)\n",
" instr_dc = run_dc.select(instrument_src, require_all=True)\n",
- " if n_trains-n_imgs != 0:\n",
+ " empty_trains = n_trains - n_imgs\n",
+ " if empty_trains != 0:\n",
" print(\n",
- " f\"\\tWARNING: {Path(run_dc.files[0].filename).name} has {n_trains-n_imgs} \" # noqa\n",
- " f\"trains with empty data out of {n_trains} trains.\")\n",
- "\n",
+ " f\"\\tWARNING: {mod} has {empty_trains} trains with empty data out of {n_trains} trains at \" # noqa\n",
+ " f\"{Path(run_dc[instrument_src, 'data.adc'].files[0].filename).parent}.\")\n",
" if max_trains > 0:\n",
" n_imgs = min(n_imgs, max_trains)\n",
- " print(f\"Processing {n_imgs} images based on the given max_trains: {max_trains}.\") # noqa\n",
+ " print(f\"Processing {n_imgs} images.\")\n",
" # Select only requested number of images to process darks.\n",
" instr_dc = instr_dc.select_trains(np.s_[:n_imgs])\n",
"\n",
@@ -236,20 +309,28 @@
" \" Not enough data to process darks.\")\n",
"\n",
" images = np.transpose(\n",
- " instr_dc[instrument_src, \"data.adc\"].ndarray(), (3, 2, 1, 0))\n",
+ " instr_dc[instrument_src, \"data.adc\"].ndarray().astype(np.float64), (3, 2, 1, 0))\n",
" acelltable = np.transpose(instr_dc[instrument_src, \"data.memoryCell\"].ndarray())\n",
+ " gain_vals = np.transpose(\n",
+ " instr_dc[instrument_src, \"data.gain\"].ndarray(), (3, 2, 1, 0))\n",
+ "\n",
+ " # define gain value as saved in raw gain map\n",
+ " raw_g = 3 if gain == 2 else gain\n",
"\n",
" if memory_cells == 1:\n",
" acelltable -= sc_start\n",
- " if gain > 0 and memory_cells == 16:\n",
- " # 255 is used as the detector sets 255 as well for\n",
- " # cell images identified as bad.\n",
- " acelltable[1:] = 255 \n",
+ " # Only for dynamic medium and low gain runs [forceswitchg1, forceswitchg2] in burst mode.\n",
+ "\n",
+ " if gain_mode == 0 and gain > 0 and memory_cells == 16:\n",
+ " # 255 similar to the receiver which uses the 255\n",
+ " # value to indicate a cell without an image.\n",
+ " # image shape for forceswitchg1 and forceswitchg2 = (1024, 512, 2, trains)\n",
+ " # compared to expected shape of (1024, 512, 16, trains) for high gain run.\n",
+ " acelltable[1:] = 255\n",
+ "\n",
+ " # Calculate offset and noise maps\n",
+ " context.map(process_cell, range(memory_cells))\n",
"\n",
- " for cell in range(memory_cells):\n",
- " thiscell = images[..., acelltable == cell]\n",
- " noise_map[mod][..., cell, gain] = np.std(thiscell, axis=2)\n",
- " offset_map[mod][..., cell, gain] = np.mean(thiscell, axis=2)\n",
" step_timer.done_step(f'Creating Offset and noise constants for a module.')"
]
},
@@ -371,11 +452,12 @@
"outputs": [],
"source": [
"def print_bp_entry(bp):\n",
- " print(\"{:<30s} {:032b}\".format(bp.name, bp.value))\n",
+ " print(\"{:<30s} {:032b} -> {}\".format(bp.name, bp.value, int(bp.value)))\n",
"\n",
"print_bp_entry(BadPixels.OFFSET_OUT_OF_THRESHOLD)\n",
"print_bp_entry(BadPixels.NOISE_OUT_OF_THRESHOLD)\n",
"print_bp_entry(BadPixels.OFFSET_NOISE_EVAL_ERROR)\n",
+ "print_bp_entry(BadPixels.WRONG_GAIN_VALUE)\n",
"\n",
"def eval_bpidx(d):\n",
"\n",
@@ -395,13 +477,12 @@
"outputs": [],
"source": [
"step_timer.start()\n",
- "bad_pixels_map = dict()\n",
"\n",
"for mod in karabo_da:\n",
- " bad_pixels_map[mod] = np.zeros(noise_map[mod].shape, np.uint32)\n",
+ " display(Markdown(f\"### Badpixels for module {mod}:\"))\n",
" offset_abs_threshold = np.array(offset_abs_threshold)\n",
"\n",
- " bad_pixels_map[mod][eval_bpidx(offset_map[mod])] = BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
+ " bad_pixels_map[mod][eval_bpidx(offset_map[mod])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
"\n",
" bad_pixels_map[mod][~np.isfinite(offset_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
"\n",
@@ -420,7 +501,8 @@
" x_label=\"Column\",\n",
" lut_label=f\"Badpixels {g_name[g_idx]} [ADCu]\",\n",
" aspect=1.,\n",
- " vmin=0, title=f'G{g_idx} Bad pixel map - Cell {cell:02d} - Module {mod}')\n",
+ " vmin=0, vmax=5,\n",
+ " title=f'G{g_idx} Bad pixel map - Cell {cell:02d} - Module {mod}')\n",
"step_timer.done_step(f'Creating bad pixels constant and plotting it for a module.')"
]
},
@@ -435,7 +517,9 @@
" memory_cells=memory_cells,\n",
" bias_voltage=bias_voltage,\n",
" integration_time=integration_time,\n",
- " gain_setting=gain_setting)\n",
+ " gain_setting=gain_setting,\n",
+ " gain_mode=gain_mode,\n",
+ ")\n",
"\n",
"db_modules = get_pdu_from_db(\n",
" karabo_id=karabo_id,\n",
@@ -511,7 +595,19 @@
" f\"• Memory cells: {memory_cells}\\n\"\n",
" f\"• Integration time: {integration_time}\\n\"\n",
" f\"• Gain setting: {gain_setting}\\n\"\n",
- " f\"• Creation time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\") # noqa"
+ " f\"• Gain mode: {gain_mode}\\n\"\n",
+ " f\"• Creation time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\") # noqa\n",
+ "step_timer.done_step(\"Injecting constants.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"Total processing time {step_timer.timespan():.01f} s\")\n",
+ "step_timer.print_summary()"
]
}
],
@@ -531,7 +627,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.11"
+ "version": "3.8.12"
}
},
"nbformat": 4,
diff --git a/setup.py b/setup.py
index 926dde6e6b04e67518b2151e5e596e04fd54f4c7..60090e3420b1fdcbd78e5bc250c44452ab17bab2 100644
--- a/setup.py
+++ b/setup.py
@@ -93,7 +93,7 @@ install_requires = [
if "readthedocs.org" not in sys.executable:
install_requires += [
- "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.0.9", # noqa
+ "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.1.0", # noqa
"XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.7.0", # noqa
]
diff --git a/src/cal_tools/enums.py b/src/cal_tools/enums.py
index e02d59994925aa5530d265ff944839fb160c9613..a9bfc0182f4972c41259992b1e31cd50b04fba7d 100644
--- a/src/cal_tools/enums.py
+++ b/src/cal_tools/enums.py
@@ -5,28 +5,28 @@ class BadPixels(IntFlag):
""" The European XFEL Bad Pixel Encoding
"""
- OFFSET_OUT_OF_THRESHOLD = 0b000000000000000000001 # bit 1
- NOISE_OUT_OF_THRESHOLD = 0b000000000000000000010 # bit 2
- OFFSET_NOISE_EVAL_ERROR = 0b000000000000000000100 # bit 3
- NO_DARK_DATA = 0b000000000000000001000 # bit 4
- CI_GAIN_OF_OF_THRESHOLD = 0b000000000000000010000 # bit 5
- CI_LINEAR_DEVIATION = 0b000000000000000100000 # bit 6
- CI_EVAL_ERROR = 0b000000000000001000000 # bit 7
- FF_GAIN_EVAL_ERROR = 0b000000000000010000000 # bit 8
- FF_GAIN_DEVIATION = 0b000000000000100000000 # bit 9
- FF_NO_ENTRIES = 0b000000000001000000000 # bit 10
- CI2_EVAL_ERROR = 0b000000000010000000000 # bit 11
- VALUE_IS_NAN = 0b000000000100000000000 # bit 12
- VALUE_OUT_OF_RANGE = 0b000000001000000000000 # bit 13
- GAIN_THRESHOLDING_ERROR = 0b000000010000000000000 # bit 14
- DATA_STD_IS_ZERO = 0b000000100000000000000 # bit 15
- ASIC_STD_BELOW_NOISE = 0b000001000000000000000 # bit 16
- INTERPOLATED = 0b000010000000000000000 # bit 17
- NOISY_ADC = 0b000100000000000000000 # bit 18
- OVERSCAN = 0b001000000000000000000 # bit 19
- NON_SENSITIVE = 0b010000000000000000000 # bit 20
- NON_LIN_RESPONSE_REGION = 0b100000000000000000000 # bit 21
-
+ OFFSET_OUT_OF_THRESHOLD = 1 << 0 # bit 1
+ NOISE_OUT_OF_THRESHOLD = 1 << 1 # bit 2
+ OFFSET_NOISE_EVAL_ERROR = 1 << 2 # bit 3
+ NO_DARK_DATA = 1 << 3 # bit 4
+ CI_GAIN_OF_OF_THRESHOLD = 1 << 4 # bit 5
+ CI_LINEAR_DEVIATION = 1 << 5 # bit 6
+ CI_EVAL_ERROR = 1 << 6 # bit 7
+ FF_GAIN_EVAL_ERROR = 1 << 7 # bit 8
+ FF_GAIN_DEVIATION = 1 << 8 # bit 9
+ FF_NO_ENTRIES = 1 << 9 # bit 10
+ CI2_EVAL_ERROR = 1 << 10 # bit 11
+ VALUE_IS_NAN = 1 << 11 # bit 12
+ VALUE_OUT_OF_RANGE = 1 << 12 # bit 13
+ GAIN_THRESHOLDING_ERROR = 1 << 13 # bit 14
+ DATA_STD_IS_ZERO = 1 << 14 # bit 15
+ ASIC_STD_BELOW_NOISE = 1 << 15 # bit 16
+ INTERPOLATED = 1 << 16 # bit 17
+ NOISY_ADC = 1 << 17 # bit 18
+ OVERSCAN = 1 << 18 # bit 19
+ NON_SENSITIVE = 1 << 19 # bit 20
+ NON_LIN_RESPONSE_REGION = 1 << 20 # bit 21
+ WRONG_GAIN_VALUE = 1 << 21 # bit 22
class SnowResolution(Enum):
""" An Enum specifying how to resolve snowy pixels
@@ -42,3 +42,13 @@ class AgipdGainMode(IntEnum):
FIXED_HIGH_GAIN = 1
FIXED_MEDIUM_GAIN = 2
FIXED_LOW_GAIN = 3
+
+
+class JungfrauSettings(Enum):
+ """ Jungfrau run gain settings."""
+ DYNAMIC_GAIN = "dynamicgain"
+ DYNAMIC_GAIN_HG0 = "dynamichg0"
+ FIX_GAIN_1 = "fixgain1"
+ FIX_GAIN_2 = "fixgain2"
+ FORCE_SWITCH_HG1 = "forceswitchg1"
+ FORCE_SWITCH_HG2 = "forceswitchg2"
diff --git a/src/cal_tools/jungfraulib.py b/src/cal_tools/jungfraulib.py
index ec04f2414b8fad34482e05b78e7ff2e0b2e9b96e..402888dfa58068ccd173800834dfd165c206fcdc 100644
--- a/src/cal_tools/jungfraulib.py
+++ b/src/cal_tools/jungfraulib.py
@@ -1,3 +1,19 @@
+from typing import Tuple
+
+from cal_tools.enums import JungfrauSettings
+
+
+def _get_settings(run_dc, ctrl_src) -> str:
+ try:
+ return(JungfrauSettings(
+ run_dc.get_run_value(ctrl_src, "settings.value")))
+ except KeyError:
+ print(
+ "WARNING: \'settings.value\' key is not available at "
+ f"{run_dc.select(ctrl_src).files[0].filename},\n")
+ return
+
+
class JungfrauCtrl():
def __init__(
self,
@@ -7,37 +23,40 @@ class JungfrauCtrl():
"""Read slow data from RUN source.
:param run_dir: EXtra-data RunDirectory DataCollection object.
- :param karabo_id_control: Karabo ID for control h5file with slow data.
+ :param ctrl_src: Control source name for accessing run slow data.
"""
self.run_dc = run_dc
self.ctrl_src = ctrl_src
+ self.run_settings = _get_settings(run_dc, ctrl_src)
- def get_memory_cells(self):
+ def get_memory_cells(self) -> Tuple[int, int]:
n_storage_cells = int(self.run_dc.get_run_value(
self.ctrl_src, "storageCells.value")) + 1
storage_cell_st = int(self.run_dc.get_run_value(
self.ctrl_src, "storageCellStart.value"))
return n_storage_cells, storage_cell_st
- def get_bias_voltage(self):
+ def get_bias_voltage(self) -> int:
return(int(self.run_dc.get_run_value(
self.ctrl_src, "vHighVoltage.value")[0]))
- def get_integration_time(self):
+ def get_integration_time(self) -> float:
return(float(self.run_dc.get_run_value(
self.ctrl_src, "exposureTime.value")) * 1e6)
- def get_gain_setting(self):
- try:
- gain_str = str(self.run_dc.get_run_value(
- self.ctrl_src, "settings.value"))
- except KeyError:
- print(
- "gain_setting is not available for h5 ctrl path "
- f"/RUN/{self.karabo_id_control}/DET/CONTROL/settings/value,\n"
- "WARNING: Setting gain_setting to 0,"
- "assuming that this is an old run.\n"
- )
- gain_str = "KeyError"
- gain_setting = 1 if gain_str == 'dynamichg0' else 0 # 'dynamicgain'
- return gain_str, gain_setting
\ No newline at end of file
+ def get_gain_setting(self) -> int:
+ if self.run_settings == JungfrauSettings.DYNAMIC_GAIN_HG0:
+ gain_setting = 1
+ else: # JungfrauSettings.DYNAMIC_GAIN
+ gain_setting = 0
+ if self.run_settings != JungfrauSettings.DYNAMIC_GAIN:
+ print(
+ "WARNING: Setting gain_setting to 0, "
+ "assuming that this is an old run.\n")
+ return gain_setting
+
+ def get_gain_mode(self) -> int:
+ if self.run_settings in [JungfrauSettings.FIX_GAIN_1, JungfrauSettings.FIX_GAIN_2]: # noqa
+ return 1
+ else:
+ return 0