diff --git a/notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb b/notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb
index 0a31e9e7afad8bf828ff54f6ef8c3742ca09aa37..51d829faa2062ffb78129423b1689e6c043357ee 100644
--- a/notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb
+++ b/notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb
@@ -4,7 +4,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "# Shimadzu HPVX2 Characterization of dark and flat field\n",
+ "# Characterization of dark and flat field for Dynamic Flat Field correction\n",
"\n",
"Author: Egor Sobolev\n",
"\n",
@@ -17,21 +17,24 @@
"metadata": {},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/SPB/202121/p002919/raw/\" # input folder, required\n",
+ "in_folder = \"/gpfs/exfel/exp/SPB/202430/p900425/raw\" # input folder, required\n",
"out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu' # output folder, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
- "dark_run = 59 # which run to read data from, required\n",
- "flat_run = 40 # which run to read\n",
+ "dark_run = 1 # which run to read data from, required\n",
+ "flat_run = 2 # which run to read\n",
"\n",
"# Data files parameters.\n",
- "karabo_da = ['HPVX01'] # data aggregators\n",
- "karabo_id = \"SPB_EHD_HPVX2_2\" # karabo prefix of Shimadzu HPV-X2 devices\n",
+ "karabo_da = ['HPVX01/1', 'HPVX01/2'] # data aggregators\n",
+ "karabo_id = \"SPB_EHD_MIC\" # karabo prefix of Shimadzu HPV-X2 devices\n",
"\n",
"#receiver_id = \"PNCCD_FMT-0\" # inset for receiver devices\n",
"#path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
- "instrument_source_template = '{}/CAM/CAMERA:daqOutput' # data source path in h5file. Template filled with karabo_id and receiver_id\n",
+ "instrument_source_template = 'SPB_EHD_MIC/CAM/HPVX2_{module}:daqOutput' # data source path in h5file.\n",
+ "#instrument_source_template = 'SPB_EHD_HPVX2_{module}/CAM/CAMERA:daqOutput'\n",
"image_key = \"data.image.pixels\" # image data key in Karabo or exdf notation\n",
"\n",
+ "db_module_template = \"Shimadzu_HPVX2_{}\"\n",
+ "\n",
"# Database access parameters.\n",
"use_dir_creation_date = True # use dir creation date as data production reference date\n",
"cal_db_interface = \"tcp://max-exfl-cal001:8021\" # calibration DB interface to use\n",
@@ -40,7 +43,7 @@
"local_output = True # if True, the notebook saves dark constants locally\n",
"creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. 2019-07-04 11:02:41.00\n",
"\n",
- "n_components = 50"
+ "n_components = 50 # Number of principal components to compute"
]
},
{
@@ -58,6 +61,7 @@
"import time\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
+ "from IPython.display import display, Markdown\n",
"\n",
"from extra_data import RunDirectory\n",
"\n",
@@ -99,14 +103,29 @@
"report = get_report(metadata_folder)\n",
"cal_db_interface = get_random_db_interface(cal_db_interface)\n",
"print(f'Calibration database interface: {cal_db_interface}')\n",
- "\n",
- "instrument = karabo_id.split(\"_\")[0]\n",
- "source = instrument_source_template.format(karabo_id)\n",
+ "print()\n",
+ "\n",
+ "instrument, part, component = karabo_id.split('_')\n",
+ "\n",
+ "sources = {}\n",
+ "source_to_db = {}\n",
+ "print(\"Sources:\")\n",
+ "for da in karabo_da:\n",
+ " aggr, _, module = da.partition('/')\n",
+ " source_name = instrument_source_template.format(\n",
+ " instrument=instrument, part=part, component=component,\n",
+ " module=module\n",
+ " )\n",
+ " sources[source_name] = aggr\n",
+ " source_to_db[source_name] = db_module_template.format(module)\n",
+ " print('-', source_name)\n",
+ "print()\n",
"\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
"\n",
- "step_timer = StepTimer()"
+ "step_timer = StepTimer()\n",
+ "constants = {}"
]
},
{
@@ -122,41 +141,36 @@
"metadata": {},
"outputs": [],
"source": [
- "step_timer.start()\n",
- "\n",
- "dark_dc = RunDirectory(f\"{in_folder}/r{dark_run:04d}\")\n",
- "dark_dc = dark_dc.select([(source, image_key)])\n",
- "key_data = dark_dc[source][image_key]\n",
- "\n",
- "images_dark = key_data.ndarray()\n",
- "ntrain, npulse, ny, nx = images_dark.shape\n",
- "\n",
- "print(f\"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})\")\n",
- "print(f\"Image size: {ny} x {nx} px\")\n",
- "step_timer.done_step(\"Read dark images\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "step_timer.start()\n",
- "dark = dffc.process_dark(images_dark)\n",
- "step_timer.done_step(\"Process dark images\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "step_timer.start()\n",
- "plot_camera_image(dark)\n",
- "plt.show()\n",
- "step_timer.done_step(\"Draw offset map\")"
+ "for source, aggr in sources.items():\n",
+ " display(Markdown(f\"## {source}\"))\n",
+ "\n",
+ " # read\n",
+ " step_timer.start()\n",
+ " dark_dc = RunDirectory(f\"{in_folder}/r{dark_run:04d}\",\n",
+ " include=f\"RAW-R{dark_run:04d}-{aggr}-S*.h5\")\n",
+ " dark_dc = dark_dc.select([(source, image_key)])\n",
+ " key_data = dark_dc[source][image_key]\n",
+ "\n",
+ " images_dark = key_data.ndarray()\n",
+ " ntrain, npulse, ny, nx = images_dark.shape\n",
+ "\n",
+ " print(f\"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})\")\n",
+ " print(f\"Image size: {ny} x {nx} px\")\n",
+ " step_timer.done_step(\"Read dark images\")\n",
+ "\n",
+ " # process\n",
+ " step_timer.start()\n",
+ " dark = dffc.process_dark(images_dark)\n",
+ " module_constants = constants.setdefault(source, {})\n",
+ " module_constants[\"Offset\"] = dark\n",
+ " step_timer.done_step(\"Process dark images\")\n",
+ " display()\n",
+ "\n",
+ " # draw plots\n",
+ " step_timer.start()\n",
+ " plot_camera_image(dark)\n",
+ " plt.show()\n",
+ " step_timer.done_step(\"Draw offsets\")"
]
},
{
@@ -169,93 +183,56 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "step_timer.start()\n",
- "\n",
- "flat_dc = RunDirectory(f\"{in_folder}/r{flat_run:04d}\")\n",
- "flat_dc = flat_dc.select([(source, image_key)])\n",
- "key_data = flat_dc[source][image_key]\n",
- "\n",
- "images_flat = key_data.ndarray()\n",
- "ntrain, npulse, ny, nx = images_flat.shape\n",
- "\n",
- "print(f\"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})\")\n",
- "print(f\"Image size: {ny} x {nx} px\")\n",
- "step_timer.done_step(\"Read flat-field images\")\n",
- "\n",
- "tm0 = time.monotonic()\n",
- "tm_cm = time.monotonic() - tm0\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "step_timer.start()\n",
- "flat, components, explained_variance_ratio = dffc.process_flat(\n",
- " images_flat, dark, n_components)\n",
- "step_timer.done_step(\"Process flat-field images\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Average flat-field"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "step_timer.start()\n",
- "plot_camera_image(flat)\n",
- "plt.show()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Explained variance ratio"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "fig, ax = plt.subplots(1, 1, figsize=(10,4), tight_layout=True)\n",
- "ax.semilogy(explained_variance_ratio, 'o')\n",
- "ax.set_xticks(np.arange(len(explained_variance_ratio)))\n",
- "ax.set_xlabel(\"Component no.\")\n",
- "ax.set_ylabel(\"Variance fraction\")\n",
- "plt.show()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# The first principal components (up to 20)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
+ "metadata": {
+ "scrolled": false
+ },
"outputs": [],
"source": [
- "plot_images(components[:20], figsize=(13, 8))\n",
- "plt.show()\n",
- "step_timer.done_step(\"Draw flat-field map and components\")"
+ "for source, aggr in sources.items():\n",
+ " display(Markdown(f\"## {source}\"))\n",
+ "\n",
+ " # read\n",
+ " step_timer.start()\n",
+ " flat_dc = RunDirectory(f\"{in_folder}/r{flat_run:04d}\",\n",
+ " include=f\"RAW-R{flat_run:04d}-{aggr}-S*.h5\")\n",
+ " flat_dc = flat_dc.select([(source, image_key)])\n",
+ " key_data = flat_dc[source][image_key]\n",
+ "\n",
+ " images_flat = key_data.ndarray()\n",
+ " ntrain, npulse, ny, nx = images_flat.shape\n",
+ "\n",
+ " print(f\"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})\")\n",
+ " print(f\"Image size: {ny} x {nx} px\")\n",
+ " step_timer.done_step(\"Read flat-field images\")\n",
+ "\n",
+ " # process\n",
+ " step_timer.start()\n",
+ " flat, components, explained_variance_ratio = dffc.process_flat(\n",
+ " images_flat, dark, n_components)\n",
+ "\n",
+ " module_constants = constants.setdefault(source, {})\n",
+ " module_constants[\"DynamicFF\"] = np.concatenate([flat[None, ...], components])\n",
+ " step_timer.done_step(\"Process flat-field images\")\n",
+ "\n",
+ " # draw plots\n",
+ " step_timer.start()\n",
+ " display(Markdown(\"### Average flat-field\"))\n",
+ " plot_camera_image(flat)\n",
+ " plt.show()\n",
+ "\n",
+ " display(Markdown(\"### Explained variance ratio\"))\n",
+ " fig, ax = plt.subplots(1, 1, figsize=(10,4), tight_layout=True)\n",
+ " ax.semilogy(explained_variance_ratio, 'o')\n",
+ " ax.set_xticks(np.arange(len(explained_variance_ratio)))\n",
+ " ax.set_xlabel(\"Component no.\")\n",
+ " ax.set_ylabel(\"Variance fraction\")\n",
+ " plt.show()\n",
+ "\n",
+ " display(Markdown(\"### The first principal components (up to 20)\"))\n",
+ " plot_images(components[:20], figsize=(13, 8))\n",
+ " plt.show()\n",
+ "\n",
+ " step_timer.done_step(\"Draw flat-field\")"
]
},
{
@@ -276,45 +253,29 @@
"# Output Folder Creation:\n",
"os.makedirs(out_folder, exist_ok=True)\n",
"\n",
- "db_module = \"SHIMADZU_HPVX2_M001\"\n",
- "\n",
- "constant_name = \"Offset\"\n",
- "\n",
- "conditions = {\n",
- " 'Memory cells': {'value': 128},\n",
- " 'Pixels X': {'value': flat.shape[1]},\n",
- " 'Pixels Y': {'value': flat.shape[0]},\n",
- " 'FF components': {'value': components.shape[0]}\n",
- "}\n",
- "\n",
- "data_to_store = {\n",
- " 'condition': conditions,\n",
- " 'db_module': db_module,\n",
- " 'karabo_id': karabo_id,\n",
- " 'constant': constant_name,\n",
- " 'data': dark,\n",
- " 'creation_time': creation_time.replace(microsecond=0),\n",
- " 'file_loc': file_loc,\n",
- " 'report': report,\n",
- "}\n",
- "\n",
- "ofile = f\"{out_folder}/const_{constant_name}_{db_module}.h5\"\n",
- "if os.path.isfile(ofile):\n",
- " print(f'File {ofile} already exists and will be overwritten')\n",
- "save_dict_to_hdf5(data_to_store, ofile)\n",
- "\n",
- "\n",
- "constant_name = \"ComponentsFF\"\n",
- "\n",
- "data_to_store.update({\n",
- " 'constant': constant_name,\n",
- " 'data': np.concatenate([flat[None, ...], components]),\n",
- "})\n",
- "\n",
- "ofile = f\"{out_folder}/const_{constant_name}_{db_module}.h5\"\n",
- "if os.path.isfile(ofile):\n",
- " print(f'File {ofile} already exists and will be overwritten')\n",
- "save_dict_to_hdf5(data_to_store, ofile)\n",
+ "for source, module_constants in constants.items():\n",
+ " for constant_name, data in module_constants.items():\n",
+ " db_module = source_to_db[source]\n",
+ "\n",
+ " conditions = {\n",
+ " 'Frame Size': {'value': 1.0},\n",
+ " }\n",
+ "\n",
+ " data_to_store = {\n",
+ " 'condition': conditions,\n",
+ " 'db_module': db_module,\n",
+ " 'karabo_id': karabo_id,\n",
+ " 'constant': constant_name,\n",
+ " 'data': data,\n",
+ " 'creation_time': creation_time.replace(microsecond=0),\n",
+ " 'file_loc': file_loc,\n",
+ " 'report': report,\n",
+ " }\n",
+ "\n",
+ " ofile = f\"{out_folder}/const_{constant_name}_{db_module}.h5\"\n",
+ " if os.path.isfile(ofile):\n",
+ " print(f'File {ofile} already exists and will be overwritten')\n",
+ " save_dict_to_hdf5(data_to_store, ofile)\n",
"\n",
"step_timer.done_step(\"Storing calibration constants\")"
]