diff --git a/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
index ee6e5c160964ce530ecc99fae80ff6b7ea362039..fda112e5d0e08a1edafe0023adab63a805b8f710 100644
--- a/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
@@ -9,10 +9,10 @@
"\n",
"Author: European XFEL Detector Group, Version: 1.0\n",
"\n",
- "The following is a processing for offset, noise, and Badpixels maps using dark images taken with Gotthard2 detector.\n",
+ "The following is a processing for the dark constants (`Offset`, `Noise`, and `BadPixelsDark`) maps using dark images taken with Gotthard2 detector (GH2 50um or 25um).\n",
"All constants are evaluated per strip, per pulse, and per memory cell. The maps are calculated for each gain stage that is acquired in 3 separate runs.\n",
"\n",
- "The three maps (calibration constants) can be injected to the database and stored locally."
+ "The three maps are of shape (stripes, cells, gains): (1280, 2, 3). They can be injected to the database (`db_output`) and/or stored locally (`local_output`)."
]
},
{
@@ -24,6 +24,7 @@
"source": [
"in_folder = \"/gpfs/exfel/exp/FXE/202231/p900298/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/gotthard2/darks\" # the folder to output to, required\n",
+ "metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"run_high = 7 # run number for G0 dark run, required\n",
"run_med = 8 # run number for G1 dark run, required\n",
"run_low = 9 # run number for G2 dark run, required\n",
@@ -33,15 +34,13 @@
"karabo_da = [\"GH201\"] # data aggregators\n",
"receiver_template = \"RECEIVER\" # receiver template used to read INSTRUMENT keys.\n",
"control_template = \"CONTROL\" # control template used to read CONTROL keys.\n",
- "instrument_source_template = '{}/DET/{}:daqOutput' # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput' # noqa\n",
"ctrl_source_template = '{}/DET/{}' # template for control source name (filled with karabo_id_control)\n",
"karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n",
"\n",
"# Parameters for the calibration database.\n",
- "use_dir_creation_date = True\n",
"cal_db_interface = \"tcp://max-exfl-cal001:8020\" # calibration DB interface to use\n",
"cal_db_timeout = 300000 # timeout on caldb requests\n",
- "overwrite_creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n",
+ "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
"db_output = False # Output constants to the calibration database\n",
"local_output = True # Output constants locally\n",
"\n",
@@ -68,7 +67,6 @@
"metadata": {},
"outputs": [],
"source": [
- "import datetime\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import pasha as psh\n",
@@ -76,17 +74,20 @@
"from extra_data import RunDirectory\n",
"from pathlib import Path\n",
"\n",
+ "import yaml\n",
+ "from cal_tools.calcat_interface import CalCatApi\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.gotthard2 import gotthard2algs, gotthard2lib\n",
"from cal_tools.step_timing import StepTimer\n",
+ "from cal_tools.restful_config import calibration_client\n",
"from cal_tools.tools import (\n",
- " get_dir_creation_date,\n",
+ " calcat_creation_time,\n",
" get_constant_from_db_and_time,\n",
- " get_pdu_from_db,\n",
" get_report,\n",
" save_const_to_h5,\n",
" send_to_db,\n",
")\n",
+ "\n",
"from iCalibrationDB import Conditions, Constants\n",
"\n",
"%matplotlib inline"
@@ -102,33 +103,34 @@
"run_nums = [run_high, run_med, run_low]\n",
"in_folder = Path(in_folder)\n",
"out_folder = Path(out_folder)\n",
- "out_folder.mkdir(exist_ok=True)\n",
- "\n",
- "print(f\"Process modules: {karabo_da}\")\n",
+ "out_folder.mkdir(parents=True, exist_ok=True)\n",
"\n",
- "run_dc = RunDirectory(in_folder / f\"r{run_high:04d}\")\n",
- "file_loc = f\"proposal:{run_dc.run_metadata()['proposalNumber']} runs:{run_high} {run_med} {run_low}\" # noqa\n",
- "\n",
- "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
"ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)\n",
"\n",
"# Read report path to associate it later with injected constants.\n",
- "report = get_report(out_folder)\n",
- "\n",
- "\n",
- "if overwrite_creation_time:\n",
- " creation_time = datetime.datetime.strptime(\n",
- " overwrite_creation_time, \"%Y-%m-%d %H:%M:%S.%f\"\n",
- " )\n",
- "elif use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run_high)\n",
- " print(f\"Using {creation_time.isoformat()} as creation time\")\n",
+ "report = get_report(metadata_folder)\n",
"\n",
+ "# Run's creation time:\n",
+ "creation_time = calcat_creation_time(in_folder, run_high, creation_time)\n",
+ "print(f\"Creation time: {creation_time}\")\n",
"\n",
"if not karabo_id_control:\n",
" karabo_id_control = karabo_id"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c176a86f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_dc = RunDirectory(in_folder / f\"r{run_high:04d}\")\n",
+ "file_loc = f\"proposal:{run_dc.run_metadata()['proposalNumber']} runs:{run_high} {run_med} {run_low}\" # noqa\n",
+ "\n",
+ "receivers = sorted(list(run_dc.select(f'{karabo_id}/DET/{receiver_template}*').all_sources))"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -193,7 +195,33 @@
"single_photon = conditions[\"single_photon\"].pop()\n",
"print(\"Single photon: \", single_photon)\n",
"acquisition_rate = conditions[\"acquisition_rate\"].pop()\n",
- "print(\"Acquisition rate: \", acquisition_rate)"
+ "print(\"Acquisition rate: \", acquisition_rate)\n",
+ "\n",
+ "gh2_detector = g2ctrl.get_det_type()\n",
+ "print(f\"Processing {gh2_detector} Gotthard2.\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f64bc150-cfcd-4f98-83f9-a982fdacedd7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "calcat = CalCatApi(client=calibration_client())\n",
+ "detector_id = calcat.detector(karabo_id)['id']\n",
+ "pdus_by_da = calcat.physical_detector_units(detector_id, pdu_snapshot_at=creation_time)\n",
+ "da_to_pdu = {da: p['physical_name'] for (da, p) in pdus_by_da.items()}\n",
+ "\n",
+ "if karabo_da != [\"\"]:\n",
+ " # Filter DA connected to detector in CALCAT\n",
+ " karabo_da = [da for da in karabo_da if da in da_to_pdu]\n",
+ " # Exclude non selected DA from processing.\n",
+ " da_to_pdu = {da: da_to_pdu[da] for da in karabo_da}\n",
+ "else:\n",
+ " karabo_da = sorted(da_to_pdu.keys())\n",
+ "\n",
+ "print(f\"Processing {karabo_da}\")"
]
},
{
@@ -204,9 +232,7 @@
"outputs": [],
"source": [
"def specify_trains_to_process(\n",
- " img_key_data: \"extra_data.KeyData\", # noqa\n",
- " max_trains: int = 0,\n",
- " min_trains: int = 0,\n",
+ " img_key_data: \"extra_data.KeyData\",\n",
"):\n",
" \"\"\"Specify total number of trains to process.\n",
" Based on given min_trains and max_trains, if given.\n",
@@ -214,11 +240,11 @@
" Print number of trains to process and number of empty trains.\n",
" Raise ValueError if specified trains are less than min_trains.\n",
" \"\"\"\n",
- " # Specifies total number of trains to proccess.\n",
+ " # Specifies total number of trains to process.\n",
" n_trains = img_key_data.shape[0]\n",
" all_trains = len(img_key_data.train_ids)\n",
" print(\n",
- " f\"{mod} has {all_trains - n_trains} \"\n",
+ " f\"{receiver} has {all_trains - n_trains} \"\n",
" f\"trains with empty frames out of {all_trains} trains\"\n",
" )\n",
"\n",
@@ -250,15 +276,6 @@
" exposure_period=exposure_period,\n",
" acquisition_rate=acquisition_rate,\n",
" single_photon=single_photon,\n",
- ")\n",
- "\n",
- "db_modules = get_pdu_from_db(\n",
- " karabo_id=karabo_id,\n",
- " karabo_da=karabo_da,\n",
- " constant=Constants.Gotthard2.LUT(),\n",
- " condition=condition,\n",
- " cal_db_interface=cal_db_interface,\n",
- " snapshot_at=creation_time,\n",
")"
]
},
@@ -272,7 +289,7 @@
"def convert_train(wid, index, tid, d):\n",
" \"\"\"Convert a Gotthard2 train from 12bit to 10bit.\"\"\"\n",
" gotthard2algs.convert_to_10bit(\n",
- " d[instr_mod_src][\"data.adc\"], lut, data_10bit[index, ...]\n",
+ " d[receiver][\"data.adc\"], lut, data_10bit[index, ...]\n",
" )"
]
},
@@ -293,7 +310,7 @@
" np.uint16\n",
")\n",
"empty_lut = np.stack(1280 * [np.stack([empty_lut] * 2)], axis=0)\n",
- "for mod in karabo_da:\n",
+ "for mod, receiver in zip(karabo_da, receivers):\n",
"\n",
" # Retrieve LUT constant\n",
" lut, time = get_constant_from_db_and_time(\n",
@@ -308,10 +325,7 @@
" print_once=False,\n",
" )\n",
" print(f\"Retrieved LUT constant with creation-time {time}\")\n",
- " # Path to pixels ADC values\n",
- " instr_mod_src = instrument_src.format(int(mod[-2:]))\n",
"\n",
- " # TODO: Validate the final shape to store constants.\n",
" cshape = (1280, 2, 3)\n",
"\n",
" offset_map[mod] = context.alloc(shape=cshape, dtype=np.float32)\n",
@@ -320,10 +334,10 @@
"\n",
" for run_num, [gain, run_dc] in run_dcs_dict.items():\n",
" step_timer.start()\n",
- " n_trains = specify_trains_to_process(run_dc[instr_mod_src, \"data.adc\"])\n",
+ " n_trains = specify_trains_to_process(run_dc[receiver, \"data.adc\"])\n",
"\n",
" # Select requested number of trains to process.\n",
- " dc = run_dc.select(instr_mod_src, require_all=True).select_trains(\n",
+ " dc = run_dc.select(receiver, require_all=True).select_trains(\n",
" np.s_[:n_trains]\n",
" )\n",
"\n",
@@ -332,7 +346,7 @@
" step_timer.start()\n",
" # Convert 12bit data to 10bit\n",
" data_10bit = context.alloc(\n",
- " shape=dc[instr_mod_src, \"data.adc\"].shape, dtype=np.float32\n",
+ " shape=dc[receiver, \"data.adc\"].shape, dtype=np.float32\n",
" )\n",
" context.map(convert_train, dc)\n",
" step_timer.done_step(\"convert to 10bit\")\n",
@@ -360,7 +374,7 @@
" context.map(offset_noise_cell, (even_data, odd_data))\n",
"\n",
" # Split even and odd gain data.\n",
- " data_gain = dc[instr_mod_src, \"data.gain\"].ndarray()\n",
+ " data_gain = dc[receiver, \"data.gain\"].ndarray()\n",
" even_gain = data_gain[:, 20::2, :]\n",
" odd_gain = data_gain[:, 21::2, :]\n",
" raw_g = 3 if gain == 2 else gain\n",
@@ -423,7 +437,7 @@
"step_timer.start()\n",
"g_name = [\"G0\", \"G1\", \"G2\"]\n",
"\n",
- "for mod, pdu in zip(karabo_da, db_modules):\n",
+ "for mod, pdu in da_to_pdu.items():\n",
" display(Markdown(f\"### Badpixels for module {mod}:\"))\n",
"\n",
" badpixels_map[mod][\n",
@@ -436,19 +450,19 @@
" badpixels_map[mod][\n",
" ~np.isfinite(noise_map[mod])\n",
" ] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
- "\n",
- " for cell in [0, 1]:\n",
- " fig, ax = plt.subplots(figsize=(10, 5))\n",
- " for g_idx in [0, 1, 2]:\n",
- " ax.plot(badpixels_map[mod][:, cell, g_idx], label=f\"G{g_idx} Bad pixel map\")\n",
- " ax.set_xticks(np.arange(0, 1281, 80))\n",
- " ax.set_xlabel(\"Stripes #\")\n",
- " ax.set_xlabel(\"BadPixels\")\n",
- " ax.set_title(f\"Cell {cell} - Module {mod} ({pdu})\")\n",
- " ax.set_ylim([0, 5])\n",
- " ax.legend()\n",
- " pass\n",
- "step_timer.done_step(f\"Creating bad pixels constant and plotting it.\")"
+ " if not local_output:\n",
+ " for cell in [0, 1]:\n",
+ " fig, ax = plt.subplots(figsize=(10, 5))\n",
+ " for g_idx in [0, 1, 2]:\n",
+ " ax.plot(badpixels_map[mod][:, cell, g_idx], label=f\"G{g_idx} Bad pixel map\")\n",
+ " ax.set_xticks(np.arange(0, 1281, 80))\n",
+ " ax.set_xlabel(\"Stripes #\")\n",
+ " ax.set_xlabel(\"BadPixels\")\n",
+ " ax.set_title(f\"BadPixels map - Cell {cell} - Module {mod} ({pdu})\")\n",
+ " ax.set_ylim([0, 5])\n",
+ " ax.legend()\n",
+ " plt.show()\n",
+ "step_timer.done_step(f\"Creating bad pixels constant.\")"
]
},
{
@@ -458,18 +472,21 @@
"metadata": {},
"outputs": [],
"source": [
- "for mod, pdu in zip(karabo_da, db_modules):\n",
+ "if not local_output:\n",
" for cons, cname in zip([offset_map, noise_map], [\"Offset\", \"Noise\"]):\n",
- " for cell in [0, 1]:\n",
- " fig, ax = plt.subplots(figsize=(10, 5))\n",
- " for g_idx in [0, 1, 2]:\n",
- " ax.plot(cons[mod][:, cell, g_idx], label=f\"G{g_idx} {cname} map\")\n",
- "\n",
- " ax.set_xlabel(\"Stripes #\")\n",
- " ax.set_xlabel(cname)\n",
- " ax.set_title(f\"{cname} map - Cell {cell} - Module {mod} ({pdu})\")\n",
- " ax.legend()\n",
- " pass"
+ " for mod, pdu in da_to_pdu.items():\n",
+ " display(Markdown(f\"### {cname} for module {mod}:\"))\n",
+ "\n",
+ " for cell in [0, 1]:\n",
+ " fig, ax = plt.subplots(figsize=(10, 5))\n",
+ " for g_idx in [0, 1, 2]:\n",
+ " ax.plot(cons[mod][:, cell, g_idx], label=f\"G{g_idx} {cname} map\")\n",
+ "\n",
+ " ax.set_xlabel(\"Stripes #\")\n",
+ " ax.set_xlabel(cname)\n",
+ " ax.set_title(f\"{cname} map - Cell {cell} - Module {mod} ({pdu})\")\n",
+ " ax.legend()\n",
+ " plt.show()"
]
},
{
@@ -480,7 +497,7 @@
"outputs": [],
"source": [
"step_timer.start()\n",
- "for mod, db_mod in zip(karabo_da, db_modules):\n",
+ "for mod, db_mod in da_to_pdu.items():\n",
" constants = {\n",
" \"Offset\": offset_map[mod],\n",
" \"Noise\": noise_map[mod],\n",
@@ -531,6 +548,26 @@
")\n",
"step_timer.done_step(\"Injecting constants.\")"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "98ca9486",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# TODO: store old constants for comparison.\n",
+ "for mod, pdu in da_to_pdu.items():\n",
+ " mod_file = mod.replace(\"/\", \"-\")\n",
+ " with open(f\"{metadata_folder or out_folder}/module_metadata_{mod_file}.yml\", \"w\") as fd:\n",
+ " yaml.safe_dump(\n",
+ " {\n",
+ " \"module\": mod,\n",
+ " \"pdu\": pdu,\n",
+ " },\n",
+ " fd,\n",
+ " )"
+ ]
}
],
"metadata": {
diff --git a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
index 134e7e0bb729e7faf6c1509ea6d65c46156709d2..aff16f80037d928e20ea830865d763ffa636d065 100644
--- a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
@@ -5,11 +5,42 @@
"id": "bed7bd15-21d9-4735-82c1-c27c1a5e3346",
"metadata": {},
"source": [
- "# Gotthard2 Offline Correction #\n",
+ "# Gotthard2 Offline Correction\n",
"\n",
"Author: European XFEL Detector Group, Version: 1.0\n",
"\n",
- "Offline Calibration for the Gothard2 Detector"
+ "Offline Correction for Gotthard2 Detector.\n",
+ "\n",
+ "This notebook is able to correct 25um and 50um GH2 detectors using the same correction steps:\n",
+ "- Convert 12bit raw data into 10bit, offset subtraction, then multiply with gain constant.\n",
+ "\n",
+ "| Correction | constants | boolean to enable/disable |\n",
+ "|------------|-------------|-----------------------------|\n",
+ "| 12bit to 10bit | `LUTGotthard2` | |\n",
+ "| Offset | `OffsetGotthard2`|`offset_correction`|\n",
+ "| Relative gain | `RelativeGainGotthard2` + `BadPixelsFFGotthard2` |`gain_correction`|\n",
+ "\n",
+ "Beside the corrected data, a mask is stored using the badpixels constant of the same parameter conditions and time.\n",
+ "- `BadPixelsDarkGotthard2`\n",
+ "- `BadPixelsFFGotthard2`, if relative gain correction is requested.\n",
+ "\n",
+ "The correction is done per sequence file. If all selected sequence files have no images to correct the notebook will fail.\n",
+ "The same result would be reached in case the needed dark calibration constants were not retrieved for all modules and `offset_correction` is True.\n",
+ "In case one of the gain constants were not retrieved `gain_correction` is switched to False and gain correction is disabled.\n",
+ "\n",
+ "The `data` datasets stored in the RECEIVER source along with the corrected image (`adc`) and `mask` are:\n",
+ "\n",
+ " - `gain`\n",
+ "\n",
+ " - `bunchId`\n",
+ "\n",
+ " - `memoryCell`\n",
+ "\n",
+ " - `frameNumber`\n",
+ "\n",
+ " - `timestamp`\n",
+ "\n",
+ " - `trainId`"
]
},
{
@@ -19,21 +50,21 @@
"metadata": {},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/FXE/202221/p003225/raw\" # the folder to read data from, required\n",
+ "in_folder = \"/gpfs/exfel/exp/DETLAB/202330/p900326/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/gotthard2\" # the folder to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
- "run = 50 # run to process, required\n",
+ "run = 20 # run to process, required\n",
"sequences = [-1] # sequences to correct, set to [-1] for all, range allowed\n",
"sequences_per_node = 1 # number of sequence files per node if notebook executed through xfel-calibrate, set to 0 to not run SLURM parallel\n",
"\n",
"# Parameters used to access raw data.\n",
- "karabo_id = \"FXE_XAD_G2XES\" # karabo prefix of Gotthard-II devices\n",
- "karabo_da = [\"GH201\"] # data aggregators\n",
+ "karabo_id = \"DETLAB_25UM_GH2\" # karabo prefix of Gotthard-II devices\n",
+ "karabo_da = [\"\"] # data aggregators\n",
"receiver_template = \"RECEIVER\" # receiver template used to read INSTRUMENT keys.\n",
"control_template = \"CONTROL\" # control template used to read CONTROL keys.\n",
- "instrument_source_template = \"{}/DET/{}:daqOutput\" # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'\n",
"ctrl_source_template = \"{}/DET/{}\" # template for control source name (filled with karabo_id_control)\n",
"karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n",
+ "corr_data_source = \"{}/CORR/{}:daqOutput\" # Correction data source. filled with karabo_id and correction receiver\n",
"\n",
"# Parameters for calibration database.\n",
"cal_db_interface = \"tcp://max-exfl-cal001:8016#8025\" # the database interface to use.\n",
@@ -89,6 +120,7 @@
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
" write_constants_fragment,\n",
+ " map_seq_files,\n",
")\n",
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"\n",
@@ -112,44 +144,13 @@
"if not karabo_id_control:\n",
" karabo_id_control = karabo_id\n",
"\n",
- "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
"ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)\n",
"\n",
- "print(f\"Process modules: {karabo_da} for run {run}\")\n",
- "\n",
"# Run's creation time:\n",
"creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
"print(f\"Creation time: {creation_time}\")"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "b5eb816e-b5f2-44ce-9907-0273d82341b6",
- "metadata": {},
- "outputs": [],
- "source": [
- "# Select only sequence files to process for the selected detector.\n",
- "if sequences == [-1]:\n",
- " possible_patterns = list(f\"*{mod}*.h5\" for mod in karabo_da)\n",
- "else:\n",
- " possible_patterns = list(\n",
- " f\"*{mod}-S{s:05d}.h5\" for mod in karabo_da for s in sequences\n",
- " )\n",
- "\n",
- "run_folder = Path(in_folder / f\"r{run:04d}\")\n",
- "seq_files = [\n",
- " f for f in run_folder.glob(\"*.h5\") if any(f.match(p) for p in possible_patterns)\n",
- "]\n",
- "\n",
- "seq_files = sorted(seq_files)\n",
- "\n",
- "if not seq_files:\n",
- " raise IndexError(\"No sequence files available for the selected sequences.\")\n",
- "\n",
- "print(f\"Processing a total of {len(seq_files)} sequence files\")"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -167,8 +168,9 @@
"metadata": {},
"outputs": [],
"source": [
- "# Read slow data\n",
"run_dc = RunDirectory(run_folder)\n",
+ "\n",
+ "# Read slow data\n",
"g2ctrl = gotthard2lib.Gotthard2Ctrl(run_dc=run_dc, ctrl_src=ctrl_src)\n",
"\n",
"if bias_voltage == -1:\n",
@@ -182,32 +184,24 @@
"if single_photon == -1:\n",
" single_photon = g2ctrl.get_single_photon()\n",
"\n",
+ "gh2_detector = g2ctrl.get_det_type()\n",
+ "\n",
"print(\"Bias Voltage:\", bias_voltage)\n",
"print(\"Exposure Time:\", exposure_time)\n",
"print(\"Exposure Period:\", exposure_period)\n",
"print(\"Acquisition Rate:\", acquisition_rate)\n",
- "print(\"Single Photon:\", single_photon)"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "8c852392-bb19-4c40-b2ce-3b787538a92d",
- "metadata": {},
- "source": [
- "### Retrieving calibration constants"
+ "print(\"Single Photon:\", single_photon)\n",
+ "print(f\"Processing {gh2_detector} Gotthard2.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "5717d722",
+ "id": "21a8953a-8c76-475e-8f4f-b201cc25c159",
"metadata": {},
"outputs": [],
"source": [
- "da_to_pdu = {}\n",
- "# Used for old FXE (p003225) runs before adding Gotthard2 to CALCAT\n",
- "const_data = dict()\n",
- "\n",
+ "# GH2 calibration data object.\n",
"g2_cal = GOTTHARD2_CalibrationData(\n",
" detector_name=karabo_id,\n",
" sensor_bias_voltage=bias_voltage,\n",
@@ -218,16 +212,53 @@
" event_at=creation_time,\n",
" client=rest_cfg.calibration_client(),\n",
")\n",
+ "\n",
+ "da_to_pdu = None\n",
"# Keep as long as it is essential to correct\n",
"# RAW data (FXE p003225) before the data mapping was added to CALCAT.\n",
"try: # in case local constants are used with old RAW data. This can be removed in the future.\n",
- " for mod_info in g2_cal.physical_detector_units.values():\n",
- " da_to_pdu[mod_info[\"karabo_da\"]] = mod_info[\"physical_name\"]\n",
- " db_modules = [da_to_pdu[da] for da in karabo_da]\n",
+ " da_to_pdu = g2_cal.mod_to_pdu\n",
"except CalCatError as e:\n",
" print(e)\n",
" db_modules = [None] * len(karabo_da)\n",
"\n",
+ "if da_to_pdu:\n",
+ " if karabo_da == [\"\"]:\n",
+ " karabo_da = sorted(da_to_pdu.keys())\n",
+ " else:\n",
+ " # Exclude non selected DA from processing.\n",
+ " karabo_da = [da for da in karabo_da if da in da_to_pdu]\n",
+ "\n",
+ " db_modules = [da_to_pdu[da] for da in karabo_da]\n",
+ "\n",
+ "print(f\"Process modules: {db_modules} for run {run}\")\n",
+ "\n",
+ "# Create the correction receiver name.\n",
+ "receivers = sorted(list(run_dc.select(f'{karabo_id}/DET/{receiver_template}*').all_sources))\n",
+ "if gh2_detector == \"25um\": # For 25um use virtual karabo_das for CALCAT data mapping.\n",
+ " corr_receiver = receivers[0].split(\"/\")[-1].split(\":\")[0][:-2]\n",
+ "else:\n",
+ " corr_receiver = receivers[0].split(\"/\")[-1].split(\":\")[0]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8c852392-bb19-4c40-b2ce-3b787538a92d",
+ "metadata": {},
+ "source": [
+ "### Retrieving calibration constants"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5717d722",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Used for old FXE (p003225) runs before adding Gotthard2 to CALCAT\n",
+ "const_data = dict()\n",
+ "\n",
"if constants_file:\n",
" for mod in karabo_da:\n",
" const_data[mod] = dict()\n",
@@ -243,6 +274,8 @@
" constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n",
"\n",
" g2_metadata = g2_cal.metadata(calibrations=constant_names)\n",
+ " # Display retrieved calibration constants timestamps\n",
+ " g2_cal.display_markdown_retrieved_constants(metadata=g2_metadata)\n",
"\n",
" # Validate the constants availability and raise/warn correspondingly.\n",
" for mod, calibrations in g2_metadata.items():\n",
@@ -283,24 +316,48 @@
"\n",
"# Prepare constant arrays.\n",
"if not constants_file:\n",
- " # Create the mask array.\n",
- " bpix = const_data[mod].get(\"BadPixelsDarkGotthard2\")\n",
- " if bpix is None:\n",
- " bpix = np.zeros((1280, 2, 3), dtype=np.uint32)\n",
- " if const_data[mod].get(\"BadPixelsFFGotthard2\") is not None:\n",
- " bpix |= const_data[mod][\"BadPixelsFFGotthard2\"]\n",
- " const_data[mod][\"Mask\"] = bpix\n",
- "\n",
- " # Prepare empty arrays for missing constants.\n",
- " if const_data[mod].get(\"OffsetGotthard2\") is None:\n",
- " const_data[mod][\"OffsetGotthard2\"] = np.zeros(\n",
- " (1280, 2, 3), dtype=np.float32)\n",
- "\n",
- " if const_data[mod].get(\"RelativeGainGotthard2\") is None:\n",
- " const_data[mod][\"RelativeGainGotthard2\"] = np.ones(\n",
- " (1280, 2, 3), dtype=np.float32)\n",
- " const_data[mod][\"RelativeGainGotthard2\"] = const_data[mod][\"RelativeGainGotthard2\"].astype( # noqa\n",
- " np.float32, copy=False) # Old gain constants are not float32."
+ " for mod in karabo_da:\n",
+ " # Create the mask array.\n",
+ " bpix = const_data[mod].get(\"BadPixelsDarkGotthard2\")\n",
+ " if bpix is None:\n",
+ " bpix = np.zeros((1280, 2, 3), dtype=np.uint32)\n",
+ " if const_data[mod].get(\"BadPixelsFFGotthard2\") is not None:\n",
+ " bpix |= const_data[mod][\"BadPixelsFFGotthard2\"]\n",
+ " const_data[mod][\"Mask\"] = bpix\n",
+ "\n",
+ " # Prepare empty arrays for missing constants.\n",
+ " if const_data[mod].get(\"OffsetGotthard2\") is None:\n",
+ " const_data[mod][\"OffsetGotthard2\"] = np.zeros(\n",
+ " (1280, 2, 3), dtype=np.float32)\n",
+ "\n",
+ " if const_data[mod].get(\"RelativeGainGotthard2\") is None:\n",
+ " const_data[mod][\"RelativeGainGotthard2\"] = np.ones(\n",
+ " (1280, 2, 3), dtype=np.float32)\n",
+ " const_data[mod][\"RelativeGainGotthard2\"] = const_data[mod][\"RelativeGainGotthard2\"].astype( # noqa\n",
+ " np.float32, copy=False) # Old gain constants are not float32."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2c7dd0bb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "file_da = list({kda.split('/')[0] for kda in karabo_da})\n",
+ "mapped_files, total_files = map_seq_files(\n",
+ " run_folder,\n",
+ " file_da,\n",
+ " sequences,\n",
+ ")\n",
+ "# This notebook doesn't account for processing more\n",
+ "# than one file data aggregator.\n",
+ "seq_files = mapped_files[file_da[0]]\n",
+ "\n",
+ "if not len(seq_files):\n",
+ " raise IndexError(\n",
+ " \"No sequence files available to correct for the selected sequences and karabo_da.\")\n",
+ "print(f\"Processing a total of {total_files} sequence files\")"
]
},
{
@@ -327,7 +384,7 @@
" data_corr[index, ...],\n",
" mask[index, ...],\n",
" g,\n",
- " const_data[mod][\"OffsetGotthard2\"],\n",
+ " const_data[mod][\"OffsetGotthard2\"].astype(np.float32), # PSI map is in f8\n",
" const_data[mod][\"RelativeGainGotthard2\"], \n",
" const_data[mod][\"Mask\"],\n",
" apply_offset=offset_correction,\n",
@@ -342,22 +399,31 @@
"metadata": {},
"outputs": [],
"source": [
- "for mod in karabo_da:\n",
- " # This is used in case receiver template consists of\n",
- " # karabo data aggregator index. e.g. detector at DETLAB\n",
- " instr_mod_src = instrument_src.format(mod[-2:])\n",
- " data_path = \"INSTRUMENT/\" + instr_mod_src + \"/data\"\n",
- " for raw_file in seq_files:\n",
- " step_timer.start()\n",
+ "corr_data_source = corr_data_source.format(karabo_id, corr_receiver)\n",
+ "\n",
+ "for raw_file in seq_files:\n",
"\n",
- " dc = H5File(raw_file)\n",
- " out_file = out_folder / raw_file.name.replace(\"RAW\", \"CORR\")\n",
+ " out_file = out_folder / raw_file.name.replace(\"RAW\", \"CORR\")\n",
+ " # Select module INSTRUMENT sources and deselect empty trains.\n",
+ " dc = H5File(raw_file).select(receivers, require_all=True)\n",
"\n",
- " # Select module INSTRUMENT source and deselect empty trains.\n",
- " dc = dc.select(instr_mod_src, require_all=True)\n",
- " data = dc[instr_mod_src, \"data.adc\"].ndarray()\n",
- " gain = dc[instr_mod_src, \"data.gain\"].ndarray()\n",
- " step_timer.done_step(\"preparing raw data\")\n",
+ " n_trains = len(dc.train_ids)\n",
+ "\n",
+ " # Initialize GH2 data and gain arrays to store in corrected files.\n",
+ " if gh2_detector == \"25um\":\n",
+ " data_stored = np.zeros((dc[receivers[0], \"data.adc\"].shape[:2] + (1280 * 2,)), dtype=np.float32)\n",
+ " gain_stored = np.zeros((dc[receivers[0], \"data.adc\"].shape[:2] + (1280 * 2,)), dtype=np.uint8)\n",
+ " else:\n",
+ " data_stored = None\n",
+ " gain_stored = None\n",
+ "\n",
+ " for i, (receiver, mod) in enumerate(zip(receivers, karabo_da)):\n",
+ " step_timer.start()\n",
+ " print(f\"Correcting {receiver} for {raw_file}\")\n",
+ "\n",
+ " data = dc[receiver, \"data.adc\"].ndarray()\n",
+ " gain = dc[receiver, \"data.gain\"].ndarray()\n",
+ " step_timer.done_step(\"Preparing raw data\")\n",
" dshape = data.shape\n",
"\n",
" step_timer.start()\n",
@@ -366,7 +432,7 @@
" data_corr = context.alloc(shape=dshape, dtype=np.float32)\n",
" mask = context.alloc(shape=dshape, dtype=np.uint32)\n",
" context.map(correct_train, data)\n",
- " step_timer.done_step(\"Correcting one sequence file\")\n",
+ " step_timer.done_step(f\"Correcting one receiver in one sequence file\")\n",
"\n",
" step_timer.start()\n",
"\n",
@@ -375,47 +441,54 @@
" data_corr[np.isinf(data_corr)] = np.nan\n",
"\n",
" # Create CORR files and add corrected data sections.\n",
- " image_counts = dc[instrument_src, \"data.adc\"].data_counts(labelled=False)\n",
- "\n",
- " with DataFile(out_file, \"w\") as ofile:\n",
- " # Create INDEX datasets.\n",
- " ofile.create_index(dc.train_ids, from_file=dc.files[0])\n",
- " # Create METDATA datasets\n",
- " ofile.create_metadata(\n",
- " like=dc,\n",
- " sequence=dc.run_metadata()[\"sequenceNumber\"],\n",
- " instrument_channels=(f\"{instrument_src}/data\",)\n",
- " )\n",
+ " image_counts = dc[receiver, \"data.adc\"].data_counts(labelled=False)\n",
+ "\n",
+ " if gh2_detector == \"25um\":\n",
+ " data_stored[..., i::2] = data_corr.copy()\n",
+ " gain_stored[..., i::2] = gain.copy()\n",
+ " else: # \"50um\"\n",
+ " data_stored = data_corr\n",
+ " gain_stored = gain\n",
+ "\n",
+ " with DataFile(out_file, \"w\") as ofile:\n",
+ " # Create INDEX datasets.\n",
+ " ofile.create_index(dc.train_ids, from_file=dc.files[0])\n",
+ " ofile.create_metadata(\n",
+ " like=dc,\n",
+ " sequence=dc.run_metadata()[\"sequenceNumber\"],\n",
+ " instrument_channels=(f\"{corr_data_source}/data\",)\n",
+ " )\n",
"\n",
- " # Create Instrument section to later add corrected datasets.\n",
- " outp_source = ofile.create_instrument_source(instrument_src)\n",
+ " # Create Instrument section to later add corrected datasets.\n",
+ " outp_source = ofile.create_instrument_source(corr_data_source)\n",
"\n",
- " # Create count/first datasets at INDEX source.\n",
- " outp_source.create_index(data=image_counts)\n",
+ " # Create count/first datasets at INDEX source.\n",
+ " outp_source.create_index(data=image_counts)\n",
"\n",
- " # Store uncorrected trainId in the corrected file.\n",
- " outp_source.create_key(\n",
- " f\"data.trainId\", data=dc.train_ids,\n",
- " chunks=min(50, len(dc.train_ids))\n",
- " )\n",
- "\n",
- " # Create datasets with the available corrected data\n",
- " for field_name, field_data in {\n",
- " \"adc\": data_corr,\n",
- " \"gain\": gain,\n",
- " }.items():\n",
- " outp_source.create_key(\n",
- " f\"data.{field_name}\", data=field_data,\n",
- " chunks=((chunks_data,) + data_corr.shape[1:])\n",
+ " # Store uncorrected trainId in the corrected file.\n",
+ " outp_source.create_key(\n",
+ " f\"data.trainId\", data=dc.train_ids,\n",
+ " chunks=min(50, len(dc.train_ids))\n",
" )\n",
"\n",
- " for field in [\"bunchId\", \"memoryCell\", \"frameNumber\", \"timestamp\"]:\n",
+ " # Create datasets with the available corrected data\n",
+ " for field_name, field_data in {\n",
+ " \"adc\": data_stored,\n",
+ " \"gain\": gain_stored,\n",
+ " }.items():\n",
+ " outp_source.create_key(\n",
+ " f\"data.{field_name}\", data=field_data,\n",
+ " chunks=((chunks_data,) + data_corr.shape[1:])\n",
+ " )\n",
+ "\n",
+ " # For GH2 25um, the data of the second receiver is\n",
+ " # stored in the corrected file.\n",
+ " for field in [\"bunchId\", \"memoryCell\", \"frameNumber\", \"timestamp\"]:\n",
" outp_source.create_key(\n",
- " f\"data.{field}\", data=dc[instr_mod_src, f\"data.{field}\"].ndarray(),\n",
+ " f\"data.{field}\", data=dc[receiver, f\"data.{field}\"].ndarray(),\n",
" chunks=(chunks_data, data_corr.shape[1])\n",
" )\n",
- " outp_source.create_compressed_key(f\"data.mask\", data=mask)\n",
- "\n",
+ " outp_source.create_compressed_key(f\"data.mask\", data=mask)\n",
" step_timer.done_step(\"Storing data\")"
]
},
@@ -454,16 +527,25 @@
"mod_dcs = {}\n",
"first_seq_raw = seq_files[0]\n",
"first_seq_corr = out_folder / first_seq_raw.name.replace(\"RAW\", \"CORR\")\n",
- "for mod in karabo_da:\n",
- " mod_dcs[mod] = {}\n",
- " with H5File(first_seq_corr) as out_dc:\n",
- " tid, mod_dcs[mod][\"train_corr_data\"] = next(\n",
- " out_dc[instr_mod_src, \"data.adc\"].trains()\n",
- " )\n",
+ "mod_dcs[corr_data_source] = {}\n",
+ "with H5File(first_seq_corr) as out_dc:\n",
+ " tid, mod_dcs[corr_data_source][\"train_corr_data\"] = next(\n",
+ " out_dc[corr_data_source, \"data.adc\"].trains()\n",
+ " )\n",
+ "\n",
+ "if gh2_detector == \"25um\":\n",
+ " mod_dcs[corr_data_source][\"train_raw_data\"] = np.zeros((data_corr.shape[1], 1280 * 2), dtype=np.float32)\n",
+ " mod_dcs[corr_data_source][\"train_raw_gain\"] = np.zeros((data_corr.shape[1], 1280 * 2), dtype=np.uint8)\n",
+ "\n",
+ "for i, rec_mod in enumerate(receivers):\n",
" with H5File(first_seq_raw) as in_dc:\n",
- " train_dict = in_dc.train_from_id(tid)[1][instr_mod_src]\n",
- " mod_dcs[mod][\"train_raw_data\"] = train_dict[\"data.adc\"]\n",
- " mod_dcs[mod][\"train_raw_gain\"] = train_dict[\"data.gain\"]"
+ " train_dict = in_dc.train_from_id(tid)[1][rec_mod]\n",
+ " if gh2_detector == \"25um\":\n",
+ " mod_dcs[corr_data_source][\"train_raw_data\"][..., i::2] = train_dict[\"data.adc\"]\n",
+ " mod_dcs[corr_data_source][\"train_raw_gain\"][..., i::2] = train_dict[\"data.gain\"]\n",
+ " else:\n",
+ " mod_dcs[corr_data_source][\"train_raw_data\"] = train_dict[\"data.adc\"]\n",
+ " mod_dcs[corr_data_source][\"train_raw_gain\"] = train_dict[\"data.gain\"]"
]
},
{
@@ -476,28 +558,32 @@
"display(Markdown(\"### Mean RAW and CORRECTED across pulses for one train:\"))\n",
"display(Markdown(f\"Train: {tid}\"))\n",
"\n",
+ "if gh2_detector == \"50um\":\n",
+ " title = f\"{{}} data for {karabo_da} ({db_modules})\"\n",
+ "else:\n",
+ " title = f\"Interleaved {{}} data for {karabo_da} ({db_modules})\"\n",
+ "\n",
"step_timer.start()\n",
- "for mod, pdu in zip(karabo_da, db_modules):\n",
- "\n",
- " fig, ax = plt.subplots(figsize=(20, 10))\n",
- " raw_data = mod_dcs[mod][\"train_raw_data\"]\n",
- " im = ax.plot(np.mean(raw_data, axis=0))\n",
- " ax.set_title(f\"RAW module {mod} ({pdu})\")\n",
- " ax.set_xlabel(\"Strip #\", size=20)\n",
- " ax.set_ylabel(\"12-bit ADC output\", size=20)\n",
- " plt.xticks(fontsize=20)\n",
- " plt.yticks(fontsize=20)\n",
- " pass\n",
"\n",
- " fig, ax = plt.subplots(figsize=(20, 10))\n",
- " corr_data = mod_dcs[mod][\"train_corr_data\"]\n",
- " im = ax.plot(np.mean(corr_data, axis=0))\n",
- " ax.set_title(f\"CORRECTED module {mod} ({pdu})\")\n",
- " ax.set_xlabel(\"Strip #\", size=20)\n",
- " ax.set_ylabel(\"10-bit KeV. output\", size=20)\n",
- " plt.xticks(fontsize=20)\n",
- " plt.yticks(fontsize=20)\n",
- " pass\n",
+ "fig, ax = plt.subplots(figsize=(15, 15))\n",
+ "raw_data = mod_dcs[corr_data_source][\"train_raw_data\"]\n",
+ "im = ax.plot(np.mean(raw_data, axis=0))\n",
+ "ax.set_title(title.format(\"RAW\"), fontsize=20)\n",
+ "ax.set_xlabel(\"Strip #\", size=20)\n",
+ "ax.set_ylabel(\"12-bit ADC output\", size=20)\n",
+ "plt.xticks(fontsize=20)\n",
+ "plt.yticks(fontsize=20)\n",
+ "pass\n",
+ "\n",
+ "fig, ax = plt.subplots(figsize=(15, 15))\n",
+ "corr_data = mod_dcs[corr_data_source][\"train_corr_data\"]\n",
+ "im = ax.plot(np.mean(corr_data, axis=0))\n",
+ "ax.set_title(title.format(\"CORRECTED\"), fontsize=20)\n",
+ "ax.set_xlabel(\"Strip #\", size=20)\n",
+ "ax.set_ylabel(\"10-bit KeV. output\", size=20)\n",
+ "plt.xticks(fontsize=20)\n",
+ "plt.yticks(fontsize=20)\n",
+ "pass\n",
"step_timer.done_step(\"Plotting mean data\")"
]
},
@@ -511,21 +597,21 @@
"display(Markdown(f\"### RAW and CORRECTED strips across pulses for train {tid}\"))\n",
"\n",
"step_timer.start()\n",
- "for mod, pdu in zip(karabo_da, db_modules):\n",
- " for plt_data, dname in zip(\n",
- " [\"train_raw_data\", \"train_corr_data\"], [\"RAW\", \"CORRECTED\"]\n",
- " ):\n",
- " fig, ax = plt.subplots(figsize=(15, 20))\n",
- " plt.rcParams.update({\"font.size\": 20})\n",
- "\n",
- " heatmapPlot(\n",
- " mod_dcs[mod][plt_data],\n",
- " y_label=\"Pulses\",\n",
- " x_label=\"Strips\",\n",
- " title=f\"{dname} module {mod} ({pdu})\",\n",
- " use_axis=ax,\n",
- " )\n",
- " pass\n",
+ "for plt_data, dname in zip(\n",
+ " [\"train_raw_data\", \"train_corr_data\"], [\"RAW\", \"CORRECTED\"]\n",
+ "):\n",
+ " fig, ax = plt.subplots(figsize=(15, 15))\n",
+ " plt.rcParams.update({\"font.size\": 20})\n",
+ "\n",
+ " heatmapPlot(\n",
+ " mod_dcs[corr_data_source][plt_data],\n",
+ " y_label=\"Pulses\",\n",
+ " x_label=\"Strips\",\n",
+ " title=title.format(dname),\n",
+ " use_axis=ax,\n",
+ " cb_pad=0.8,\n",
+ " )\n",
+ " pass\n",
"step_timer.done_step(\"Plotting RAW and CORRECTED data for one train\")"
]
},
@@ -569,43 +655,42 @@
"source": [
"display(Markdown(\"### RAW and CORRECTED even/odd pulses for one train:\"))\n",
"display(Markdown(f\"Train: {tid}\"))\n",
- "for mod, pdu in zip(karabo_da, db_modules):\n",
- " fig, ax = plt.subplots(figsize=(20, 20))\n",
- " raw_data = mod_dcs[mod][\"train_raw_data\"]\n",
- " corr_data = mod_dcs[mod][\"train_corr_data\"]\n",
- "\n",
- " ax.plot(raw_data[odd_pulse], label=f\"Odd Pulse {odd_pulse}\")\n",
- " if even_pulse:\n",
- " ax.plot(raw_data[even_pulse], label=f\"Even Pulse {even_pulse}\")\n",
- "\n",
- " ax.set_title(f\"RAW module {mod} ({pdu})\")\n",
- " ax.set_xlabel(\"Strip #\", size=20)\n",
- " ax.set_ylabel(\"12-bit ADC RAW\", size=20)\n",
- " plt.xticks(fontsize=20)\n",
- " plt.yticks(fontsize=20)\n",
- " ax.legend()\n",
- " pass\n",
- "\n",
- " fig, ax = plt.subplots(figsize=(20, 20))\n",
- " ax.plot(corr_data[odd_pulse], label=f\"Odd Pulse {odd_pulse}\")\n",
- " if even_pulse:\n",
- " ax.plot(corr_data[even_pulse], label=f\"Even Pulse {even_pulse}\")\n",
- " ax.set_title(f\"CORRECTED module {mod} ({pdu})\")\n",
- " ax.set_xlabel(\"Strip #\", size=20)\n",
- " ax.set_ylabel(\"10-bit KeV CORRECTED\", size=20)\n",
- " plt.xticks(fontsize=20)\n",
- " plt.yticks(fontsize=20)\n",
- " ax.legend()\n",
- " pass\n",
+ "fig, ax = plt.subplots(figsize=(15, 15))\n",
+ "raw_data = mod_dcs[corr_data_source][\"train_raw_data\"]\n",
+ "corr_data = mod_dcs[corr_data_source][\"train_corr_data\"]\n",
+ "\n",
+ "ax.plot(raw_data[odd_pulse], label=f\"Odd Pulse {odd_pulse}\")\n",
+ "if even_pulse:\n",
+ " ax.plot(raw_data[even_pulse], label=f\"Even Pulse {even_pulse}\")\n",
+ "\n",
+ "ax.set_title(title.format(\"RAW\"), fontsize=20)\n",
+ "ax.set_xlabel(\"Strip #\", size=20)\n",
+ "ax.set_ylabel(\"12-bit ADC RAW\", size=20)\n",
+ "plt.xticks(fontsize=20)\n",
+ "plt.yticks(fontsize=20)\n",
+ "ax.legend()\n",
+ "pass\n",
+ "\n",
+ "fig, ax = plt.subplots(figsize=(15, 15))\n",
+ "ax.plot(corr_data[odd_pulse], label=f\"Odd Pulse {odd_pulse}\")\n",
+ "if even_pulse:\n",
+ " ax.plot(corr_data[even_pulse], label=f\"Even Pulse {even_pulse}\")\n",
+ "ax.set_title(title.format(\"CORRECTED\"), fontsize=20)\n",
+ "ax.set_xlabel(\"Strip #\", size=20)\n",
+ "ax.set_ylabel(\"10-bit KeV CORRECTED\", size=20)\n",
+ "plt.xticks(fontsize=20)\n",
+ "plt.yticks(fontsize=20)\n",
+ "ax.legend()\n",
+ "pass\n",
"step_timer.done_step(\"Plotting RAW and CORRECTED odd/even pulses.\")"
]
}
],
"metadata": {
"kernelspec": {
- "display_name": "cal4_venv",
+ "display_name": ".cal_venv",
"language": "python",
- "name": "cal4_venv"
+ "name": "python3"
},
"language_info": {
"codemirror_mode": {
@@ -618,11 +703,6 @@
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.11"
- },
- "vscode": {
- "interpreter": {
- "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
- }
}
},
"nbformat": 4,
diff --git a/notebooks/Gotthard2/Summary_Darks_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Summary_Darks_Gotthard2_NBC.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..756097845270727db4d08b130225d0f7ec89eb62
--- /dev/null
+++ b/notebooks/Gotthard2/Summary_Darks_Gotthard2_NBC.ipynb
@@ -0,0 +1,204 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Gotthard2 Dark Summary\n",
+ "\n",
+ "Author: European XFEL Detector Department, Version: 1.0\n",
+ "\n",
+ "Summary for process dark constants and a comparison with previously injected constants with the same conditions."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/DETLAB/202330/p900326/raw\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/gotthard2\" # path to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n",
+ "run_high = 20 # run number for G0 dark run, required\n",
+ "run_med = 21 # run number for G1 dark run, required\n",
+ "run_low = 22 # run number for G2 dark run, required\n",
+ "\n",
+ "# Parameters used to access raw data.\n",
+ "karabo_id = \"DETLAB_25UM_GH2\" # detector identifier.\n",
+ "karabo_da = [\"DA01/1\", \"DA01/2\"] # list of data aggregators, which corresponds to different JF modules. This is only needed for the detectors of one module.\n",
+ "control_template = \"CONTROL\" # control template used to read CONTROL keys.\n",
+ "ctrl_source_template = '{}/DET/{}' # template for control source name (filled with karabo_id_control)\n",
+ "karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n",
+ "\n",
+ "# Parameters to be used for injecting dark calibration constants.\n",
+ "local_output = True # Boolean indicating that local constants were stored in the out_folder\n",
+ "\n",
+ "# Skip the whole notebook if local_output is false in the preceding notebooks.\n",
+ "if not local_output:\n",
+ " print('No local constants saved. Skipping summary plots')\n",
+ " import sys\n",
+ " sys.exit(0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import warnings\n",
+ "from pathlib import Path\n",
+ "\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "import h5py\n",
+ "import matplotlib\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "import yaml\n",
+ "from extra_data import RunDirectory\n",
+ "from IPython.display import Markdown, display\n",
+ "\n",
+ "from cal_tools.gotthard2 import gotthard2lib\n",
+ "\n",
+ "matplotlib.use(\"agg\")\n",
+ "%matplotlib inline\n",
+ "\n",
+ "from cal_tools.tools import CalibrationMetadata"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "out_folder = Path(out_folder)\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if not karabo_id_control:\n",
+ " karabo_id_control = karabo_id\n",
+ "g2ctrl = gotthard2lib.Gotthard2Ctrl(\n",
+ " run_dc=RunDirectory(Path(in_folder) / f\"r{run_high:04d}\"),\n",
+ " ctrl_src=ctrl_source_template.format(karabo_id_control, control_template))\n",
+ "gh2_detector = g2ctrl.get_det_type()\n",
+ "print(f\"Processing {gh2_detector} Gotthard2.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mod_mapping = dict()\n",
+ "for fn in Path(metadata_folder or out_folder).glob(\"module_metadata_*.yml\"):\n",
+ " with fn.open(\"r\") as fd:\n",
+ " fdict = yaml.safe_load(fd)\n",
+ " mod_mapping[fdict[\"module\"].replace(\"-\", \"/\")] = fdict[\"pdu\"]\n",
+ "\n",
+ "mod_mapping = dict(sorted(mod_mapping.items(), key=lambda item: item[0]))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "plt_map = dict()\n",
+ "dark_constants = [\"Offset\", \"Noise\", \"BadPixelsDark\"]\n",
+ "if gh2_detector == \"25um\":\n",
+ " for cname in dark_constants:\n",
+ " plt_map[cname] = np.zeros(\n",
+ " (1280 * 2, 2, 3),\n",
+ " dtype=np.uint32 if cname == \"BadPixelsDark\" else np.float32\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for cname in dark_constants:\n",
+ "\n",
+ " for i, (mod, pdu) in enumerate(mod_mapping.items()):\n",
+ " module = mod.replace(\"-\", \"/\")\n",
+ "\n",
+ " with h5py.File(out_folder / f\"const_{cname}_{pdu}.h5\", 'r') as f:\n",
+ " if gh2_detector == \"25um\":\n",
+ " plt_map[cname][i::2] = f[\"data\"][()]\n",
+ " else:\n",
+ " plt_map[cname] = f[\"data\"][()]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if gh2_detector == \"50um\":\n",
+ " title = (\n",
+ " f\"{{}} data for \"\n",
+ " f\"{[f'{mod}({pdu})' for mod, pdu in mod_mapping.items()]}\"\n",
+ " )\n",
+ "else:\n",
+ " title = (\n",
+ " f\"Interleaved {{}} data for \"\n",
+ " f\"{[f'{mod}({pdu})' for mod, pdu in mod_mapping.items()]}\"\n",
+ " )\n",
+ "\n",
+ "for cname in dark_constants:\n",
+ " display(Markdown(f\"### {cname}\"))\n",
+ "\n",
+ " for cell in [0, 1]:\n",
+ " fig, ax = plt.subplots(figsize=(10, 5))\n",
+ " for g_idx in [0, 1, 2]:\n",
+ " ax.plot(plt_map[cname][:, cell, g_idx], label=f\"G{g_idx} {cname} map\")\n",
+ " ax.set_xticks(\n",
+ " np.arange(0, plt_map[cname].shape[0]+1, plt_map[cname].shape[0]//16)\n",
+ " )\n",
+ " ax.set_xlabel(\"Stripes #\", fontsize=15)\n",
+ " ax.set_ylabel(\"ADU\", fontsize=15)\n",
+ " ax.set_title(title.format(f\"{cname} map - Cell {cell}\"), fontsize=15)\n",
+ " if cname == \"BadPixelsDark\":\n",
+ " ax.set_ylim([0, 5])\n",
+ " ax.legend()\n",
+ " plt.show()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": ".cal3_venv",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ },
+ "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/src/cal_tools/calcat_interface.py b/src/cal_tools/calcat_interface.py
index db67afc900f82e1949dd2ef9ce37d76502a4eda4..b6fae338496ad62a1ddf33c3c6d768dcc1c8796e 100644
--- a/src/cal_tools/calcat_interface.py
+++ b/src/cal_tools/calcat_interface.py
@@ -1361,7 +1361,8 @@ class GOTTHARD2_CalibrationData(CalibrationData):
"""Calibration data for the Gotthard II detector."""
calibrations = {
- "LUTGotthard2" "OffsetGotthard2",
+ "LUTGotthard2",
+ "OffsetGotthard2",
"NoiseGotthard2",
"BadPixelsDarkGotthard2",
"RelativeGainGotthard2",
diff --git a/src/cal_tools/gotthard2/gotthard2lib.py b/src/cal_tools/gotthard2/gotthard2lib.py
index 2405ddf08a1cd2c9099890009850adca9b5f3381..cd37535e4c69cad1f1cd0ebb5f3f1bed4f3280bd 100644
--- a/src/cal_tools/gotthard2/gotthard2lib.py
+++ b/src/cal_tools/gotthard2/gotthard2lib.py
@@ -41,3 +41,14 @@ class Gotthard2Ctrl():
if "singlePhoton.value" in self.run_dc.keys_for_source(self.ctrl_src):
return bool(
self.run_dc[self.ctrl_src, "singlePhoton"].as_single_value())
+
+ def get_det_type(self):
+ """GH2 rxHostname is a vector of bytes objects.
+ GH2 25um has two host names unlike 50um which has one.
+
+ Returns:
+ str: return if its a 25um or 50um detector.
+ """
+
+ hostname = self.run_dc.get_run_value(self.ctrl_src, "rxHostname")
+ return "25um" if hostname[1].decode("utf-8") else "50um"
diff --git a/src/xfel_calibrate/calibrate.py b/src/xfel_calibrate/calibrate.py
index 5590e4c34f5f6c9c10d3700260adfb3c3e1ea2f2..8ffcc29aa14a3763ca110b56298c908b62ada903 100755
--- a/src/xfel_calibrate/calibrate.py
+++ b/src/xfel_calibrate/calibrate.py
@@ -112,6 +112,11 @@ def balance_sequences(in_folder: str, run: int, sequences: List[int],
elif not isinstance(karabo_da, list):
raise TypeError("Balance sequences expects `karabo_da` as a string or list.")
+ # data-mapping for LPD mini and GH2 25um uses karabo-da names like
+ # LPDMINI00/2 or DA01/2 to identify individual modules. The /2 is not
+ # part of the file name
+ karabo_da = list({kda.split('/')[0] for kda in karabo_da})
+
in_path = Path(in_folder, f"r{run:04d}")
# TODO: remove ["-1"] after karabo_da refactor
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index 3d241b27e7f1d41050ce49b28be9823611d9040d..63231b4dff9c60cf7a62a808ef06b270da51e7a3 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -214,6 +214,8 @@ notebooks = {
"DARK": {
"notebook":
"notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb",
+ "dep_notebooks": [
+ "notebooks/Gotthard2/Summary_Darks_Gotthard2_NBC.ipynb"],
"concurrency": {"parameter": "karabo_da",
"default concurrency": list(range(2)),
"cluster cores": 4},