Newer
Older
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Characterization of dark and flat field for Dynamic Flat Field correction\n",
"\n",
"Author: Egor Sobolev\n",
"\n",
"Computation of dark offsets and flat-field principal components"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"in_folder = \"/gpfs/exfel/exp/SPB/202430/p900425/raw\" # input folder, required\n",
"out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu' # output folder, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"run_high = 1 # run number in which dark data was recorded, required\n",
"run_low = 2 # run number in which flat-field data was recorded, required\n",
"operation_mode = \"TI_DynamicFF\" # Detector operation mode, optional (defaults to \"TI_DynamicFF\")\n",
"\n",
"# Data files parameters.\n",
"karabo_da = ['-1'] # data aggregators\n",
"karabo_id = \"SPB_MIC_HPVX2\" # karabo prefix of Shimadzu HPV-X2 devices\n",
"\n",
"# Database access parameters.\n",
"cal_db_interface = \"tcp://max-exfl-cal001:8021\" # calibration DB interface to use\n",
"db_output = True # if True, the notebook sends dark constants to the calibration database\n",
"local_output = True # if True, the notebook saves dark constants locally\n",
"\n",
"# Calibration constants parameters\n",
"n_components = 50 # Number of principal components of flat-field to compute (default: 50)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import datetime\n",
"import os\n",
"import warnings\n",
"from logging import warning\n",
Philipp Schmidt
committed
"from shutil import copyfile\n",
"from tempfile import NamedTemporaryFile\n",
"\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import time\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from IPython.display import display, Markdown\n",
"\n",
"from extra_data import RunDirectory\n",
"\n",
"%matplotlib inline\n",
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import (\n",
" get_dir_creation_date,\n",
" get_random_db_interface,\n",
" get_report,\n",
" save_dict_to_hdf5,\n",
" run_prop_seq_from_path,\n",
")\n",
"from cal_tools.restful_config import calibration_client\n",
"from cal_tools.shimadzu import ShimadzuHPVX2\n",
"\n",
"import dynflatfield as dffc\n",
"from dynflatfield.draw import plot_images, plot_camera_image"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"cal_db_interface = get_random_db_interface(cal_db_interface)\n",
"print(f'Calibration database interface: {cal_db_interface}')\n",
"print()\n",
"\n",
"cc = calibration_client()\n",
"pdus = cc.get_all_phy_det_units_from_detector(\n",
" {\"detector_identifier\": karabo_id})\n",
"\n",
"if not pdus[\"success\"]:\n",
" raise ValueException(\"Failed to retrieve PDUs\")\n",
"\n",
"detector_info = pdus['data'][0]['detector']\n",
"detector = ShimadzuHPVX2(detector_info[\"source_name_pattern\"])\n",
"\n",
"print(f\"Instrument {detector.instrument}\")\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"\n",
"modules = {}\n",
"for pdu_no, pdu in enumerate(pdus[\"data\"]):\n",
" db_module = pdu[\"physical_name\"]\n",
" module = pdu[\"module_number\"]\n",
" da = pdu[\"karabo_da\"]\n",
" if karabo_da[0] != \"-1\" and da not in karabo_da:\n",
" continue\n",
"\n",
" instrument_source_name = detector.instrument_source(module)\n",
" print('-', da, db_module, module, instrument_source_name)\n",
"\n",
" modules[da] = dict(\n",
" db_module=db_module,\n",
" module=module,\n",
" raw_source_name=instrument_source_name,\n",
" pdu_no=pdu_no,\n",
" )\n",
"\n",
"constants = {}\n",
"step_timer = StepTimer()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Offset map"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dark_run = run_high\n",
"dark_creation_time = get_dir_creation_date(in_folder, dark_run)\n",
"print(f\"Using {dark_creation_time} as creation time of Offset constant.\")\n",
"\n",
"for da, meta in modules.items():\n",
" source_name = detector.instrument_source(meta[\"module\"])\n",
" image_key = detector.image_key\n",
"\n",
" display(Markdown(f\"## {source_name}\"))\n",
"\n",
" # read\n",
" step_timer.start()\n",
" file_da, _, _ = da.partition('/')\n",
" dark_dc = RunDirectory(f\"{in_folder}/r{dark_run:04d}\",\n",
" include=f\"RAW-R{dark_run:04d}-{file_da}-S*.h5\")\n",
"\n",
" if source_name not in dark_dc.all_sources:\n",
" raise ValueError(f\"Could not find source {source_name} for module {da} in dark data\")\n",
" dark_dc = dark_dc.select([(source_name, image_key)])\n",
" conditions = detector.conditions(dark_dc, meta[\"module\"])\n",
"\n",
" key_data = dark_dc[source_name, image_key]\n",
" images_dark = key_data.ndarray()\n",
" ntrain, npulse, ny, nx = images_dark.shape\n",
"\n",
" print(f\"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})\")\n",
" print(f\"Image size: {ny} x {nx} px\")\n",
" step_timer.done_step(\"Read dark images\")\n",
"\n",
" # process\n",
" step_timer.start()\n",
" dark = dffc.process_dark(images_dark)\n",
"\n",
" # put results in the dict\n",
" module_constants = constants.setdefault(meta[\"db_module\"], {})\n",
" module_constants[\"Offset\"] = dict(\n",
" conditions=conditions, data=dark, pdu_no=meta[\"pdu_no\"],\n",
" creation_time=dark_creation_time\n",
" )\n",
" step_timer.done_step(\"Process dark images\")\n",
" display()\n",
"\n",
" # draw plots\n",
" step_timer.start()\n",
" plot_camera_image(dark)\n",
" plt.show()\n",
" step_timer.done_step(\"Draw offsets\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Flat-field PCA decomposition"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"flat_run = run_low\n",
"flat_creation_time = get_dir_creation_date(in_folder, flat_run)\n",
"print(f\"Using {flat_creation_time} as creation time of DynamicFF constant.\")\n",
"\n",
"for da, meta in modules.items():\n",
" source_name = detector.instrument_source(meta[\"module\"])\n",
" image_key = detector.image_key\n",
"\n",
" display(Markdown(f\"## {source_name}\"))\n",
"\n",
" # read\n",
" step_timer.start()\n",
" file_da, _, _ = da.partition('/')\n",
" flat_dc = RunDirectory(f\"{in_folder}/r{flat_run:04d}\",\n",
" include=f\"RAW-R{flat_run:04d}-{file_da}-S*.h5\")\n",
"\n",
" if source_name not in flat_dc.all_sources:\n",
" raise ValueError(f\"Could not find source {source_name} for module {da} in flatfield data\")\n",
" flat_dc = flat_dc.select([(source_name, image_key)])\n",
" conditions = detector.conditions(flat_dc, meta[\"module\"])\n",
"\n",
" dark = constants[meta[\"db_module\"]][\"Offset\"][\"data\"]\n",
" dark_conditions = constants[meta[\"db_module\"]][\"Offset\"][\"conditions\"]\n",
"\n",
" if conditions != dark_conditions:\n",
" ValueError(\"The conditions for flat-field run does not match \"\n",
" \"the dark run conditions. Skip flat-field characterization.\")\n",
"\n",
" key_data = flat_dc[source_name][image_key]\n",
" images_flat = key_data.ndarray()\n",
" ntrain, npulse, ny, nx = images_flat.shape\n",
"\n",
" print(f\"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})\")\n",
" print(f\"Image size: {ny} x {nx} px\")\n",
" step_timer.done_step(\"Read flat-field images\")\n",
"\n",
" # process\n",
" step_timer.start()\n",
" flat, components, explained_variance_ratio = dffc.process_flat(\n",
" images_flat, dark, n_components)\n",
" flat_data = np.concatenate([flat[None, ...], components])\n",
"\n",
" # put results in the dict\n",
" conditions = detector.conditions(flat_dc, meta[\"module\"])\n",
" module_constants = constants.setdefault(meta[\"db_module\"], {})\n",
" module_constants[\"DynamicFF\"] = dict(\n",
" conditions=conditions, data=flat_data, pdu_no=meta[\"pdu_no\"],\n",
" creation_time=flat_creation_time\n",
" step_timer.done_step(\"Process flat-field images\")\n",
"\n",
" # draw plots\n",
" step_timer.start()\n",
" display(Markdown(\"### Average flat-field\"))\n",
" plot_camera_image(flat)\n",
" plt.show()\n",
"\n",
" display(Markdown(\"### Explained variance ratio\"))\n",
" fig, ax = plt.subplots(1, 1, figsize=(10,4), tight_layout=True)\n",
" ax.semilogy(explained_variance_ratio, 'o')\n",
" ax.set_xticks(np.arange(len(explained_variance_ratio)))\n",
" ax.set_xlabel(\"Component no.\")\n",
" ax.set_ylabel(\"Variance fraction\")\n",
" plt.show()\n",
"\n",
" display(Markdown(\"### The first principal components (up to 20)\"))\n",
" plot_images(components[:20], figsize=(13, 8))\n",
" plt.show()\n",
"\n",
" step_timer.done_step(\"Draw flat-field\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Calibration constants"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"step_timer.start()\n",
"\n",
"# Output Folder Creation:\n",
Philipp Schmidt
committed
"if local_output:\n",
" os.makedirs(out_folder, exist_ok=True)\n",
"\n",
"def inject_ccv(in_folder, metadata_folder, runs, calibration, cond, pdu, const_input, begin_at):\n",
" print(\"* Send to db:\", const_input)\n",
" print(\" - in folder:\", in_folder)\n",
" print(\" - metadata folder:\", metadata_folder)\n",
" print(\" - runs:\", runs)\n",
" print(\" -\", calibration)\n",
" print(\" -\", cond)\n",
" print(\" -\", begin_at)\n",
"\n",
"for db_module, module_constants in constants.items():\n",
" for constant_name, constant in module_constants.items():\n",
" conditions = constant[\"conditions\"]\n",
" conditions_dict = conditions.make_dict(\n",
" conditions.calibration_types[constant_name])\n",
" data_to_store = {db_module: {constant_name: {'0': {\n",
" 'conditions': conditions_dict,\n",
" 'data': constant[\"data\"],\n",
" }}}}\n",
"\n",
Philipp Schmidt
committed
" with NamedTemporaryFile() as tempf:\n",
" save_dict_to_hdf5(data_to_store, tempf)\n",
" \n",
" if db_output:\n",
" inject_ccv(\n",
" in_folder, metadata_folder, [dark_run, flat_run],\n",
" constant_name, conditions, pdus[\"data\"][constant[\"pdu_no\"]],\n",
" ofile, constant[\"creation_time\"]\n",
" )\n",
" \n",
" if local_output:\n",
" ofile = f\"{out_folder}/const_{constant_name}_{db_module}.h5\"\n",
" \n",
" if os.path.isfile(ofile):\n",
" print(f'File {ofile} already exists and will be overwritten')\n",
" \n",
" copyfile(tempf.name, ofile)"
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(f\"Total processing time {step_timer.timespan():.01f} s\")\n",
"step_timer.print_summary()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.11"
}
},
"nbformat": 4,
"nbformat_minor": 4
}