[Generic][Shimadzu] Dynamic flat-field characterization and correction for MHz microscopy

This implements:

1. The characterization of dark and flat fields of Shimadzu HPVX2 images
2. The correction for Shimadzu HPVX2 images by applying offsets subtraction and dynamic flat-field correction.

Depends on the https://github.com/sarlotabirnsteinova/OnlineVisualization

This is needed for MHz microscopy at SPB

Features:

• Split on sequence files by 200 trains, ignoring original splitting
• Creates new source name SPB_EHD_MIC/CAM/HPVX2_1:daqOutput -> SPB_EHD_MIC/CORR/HPVX2_1:output

Corrected files structure

CORR-R0003-HPVX01-S00000.h5 ├INDEX │ ├SPB_EHD_MIC │ │ └CORR │ │ ├HPVX2_1:output │ │ │ └index_group │ │ │ ├count [uint64: 15] │ │ │ └first [uint64: 15] │ │ └HPVX2_2:output │ │ └index_group │ │ ├count [uint64: 15] │ │ └first [uint64: 15] │ ├flag [int32: 15] │ ├origin [int32: 15] │ ├timestamp [uint64: 15] │ └trainId [uint64: 15] ├INSTRUMENT │ └SPB_EHD_MIC │ └CORR │ ├HPVX2_1:output │ │ └data │ │ └image │ │ ├binning [uint64: 9 × 3] │ │ ├dimTypes [int32: 9 × 3] │ │ ├dims [uint64: 9 × 3] │ │ ├flipX [uint8: 9] │ │ ├flipY [uint8: 9] │ │ ├pixels [float32: 9 × 128 × 250 × 400] │ │ ├roiOffsets [uint64: 9 × 3] │ │ └rotation [int32: 9] │ └HPVX2_2:output │ └data │ └image │ ├binning [uint64: 10 × 3] │ ├dimTypes [int32: 10 × 3] │ ├dims [uint64: 10 × 3] │ ├flipX [uint8: 10] │ ├flipY [uint8: 10] │ ├pixels [float32: 10 × 128 × 250 × 400] │ ├roiOffsets [uint64: 10 × 3] │ └rotation [int32: 10] └METADATA ├creationDate [ASCII string: 1] ├daqLibrary [ASCII string: 1] ├dataFormatVersion [ASCII string: 1] ├dataSources │ ├dataSourceId [ASCII string: 2] │ ├deviceId [ASCII string: 2] │ └root [ASCII string: 2] ├karaboFramework [ASCII string: 1] ├proposalNumber [uint32: 1] ├runNumber [uint32: 1] ├sequenceNumber [uint32: 1] └updateDate [ASCII string: 1]

Reports:

• characterization report: dark_runs_1_2_v0.pdf
• correction report: r0003.pdf

cc @schmidtp

notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb

 %% Cell type:markdown id: tags:
 
 # Characterization of dark and flat field for Dynamic Flat Field correction
 
 Author: Egor Sobolev
 
 Computation of dark offsets and flat-field principal components
 
 %% Cell type:code id: tags:
 
 ``` python
 in_folder = "/gpfs/exfel/exp/SPB/202430/p900425/raw"  # input folder, required
 out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu'  # output folder, required
 metadata_folder = ""  # Directory containing calibration_metadata.yml when run by xfel-calibrate
-runs = [1, 2]  # list of two run numbers: dark field and flat field
+run_high = 1 # run number in which dark data was recorded, required
+run_low = 2 # run number in which flat-field data was recorded, required
+operation_mode = "TI_DynamicFF"  # Detector operation mode, optional (defaults to "TI_DynamicFF")
 
 # Data files parameters.
 karabo_da = ['-1'] # data aggregators
 karabo_id = "SPB_MIC_HPVX2" # karabo prefix of Shimadzu HPV-X2 devices
 
-#receiver_id = "PNCCD_FMT-0" # inset for receiver devices
-#path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5'  # the template to use to access data
-#instrument_source_template = 'SPB_EHD_MIC/CAM/HPVX2_{module}:daqOutput'  # data source path in h5file.
-#instrument_source_template = 'SPB_EHD_HPVX2_{module}/CAM/CAMERA:daqOutput'
-#image_key = "data.image.pixels"  # image data key in Karabo or exdf notation
-
-#db_module_template = "Shimadzu_HPVX2_{}"
-
 # Database access parameters.
-use_dir_creation_date = True  # use dir creation date as data production reference date
 cal_db_interface = "tcp://max-exfl-cal001:8021"  # calibration DB interface to use
-cal_db_timeout = 300000 # timeout on caldb requests
 db_output = True # if True, the notebook sends dark constants to the calibration database
 local_output = True # if True, the notebook saves dark constants locally
-creation_time = "" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. 2019-07-04 11:02:41.00
 
-n_components = 50  # Number of principal components to compute
+# Calibration constants parameters
+n_components = 50  # Number of principal components of flat-field to compute (default: 50)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 import datetime
 import os
 import warnings
 
 warnings.filterwarnings('ignore')
 
 import time
 import numpy as np
 import matplotlib.pyplot as plt
 from IPython.display import display, Markdown
 
 from extra_data import RunDirectory
 
 %matplotlib inline
 from cal_tools.step_timing import StepTimer
 from cal_tools.tools import (
     get_dir_creation_date,
-#    get_pdu_from_db,
     get_random_db_interface,
     get_report,
-#    save_const_to_h5,
     save_dict_to_hdf5,
-#    send_to_db,
     run_prop_seq_from_path,
 )
 from cal_tools.restful_config import calibration_client
 from cal_tools.shimadzu import ShimadzuHPVX2
 
 import dynflatfield as dffc
 from dynflatfield.draw import plot_images, plot_camera_image
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
-creation_time=None
-if use_dir_creation_date:
-    creation_time = get_dir_creation_date(in_folder, max(runs))
-
-print(f"Using {creation_time} as creation time of constant.")
-
-run, proposal, seq = run_prop_seq_from_path(in_folder)
-#file_loc = f'proposal: {prop}, runs: {dark_run} {flat_run}'
-
-# Read report path and create file location tuple to add with the injection
-#file_loc = f"proposal:{proposal} runs:" + ' '.join(str(run) for run in runs)
-
-report = get_report(metadata_folder)
 cal_db_interface = get_random_db_interface(cal_db_interface)
 print(f'Calibration database interface: {cal_db_interface}')
 print()
 
 cc = calibration_client()
 pdus = cc.get_all_phy_det_units_from_detector(
     {"detector_identifier": karabo_id})
 
 if not pdus["success"]:
     print("exception")
 
 detector_info = pdus['data'][0]['detector']
 detector = ShimadzuHPVX2(detector_info["source_name_pattern"])
 
 print(f"Instrument {detector.instrument}")
 print(f"Detector in use is {karabo_id}")
 
 modules = {}
 for pdu_no, pdu in enumerate(pdus["data"]):
     db_module = pdu["physical_name"]
     module = pdu["module_number"]
     da = pdu["karabo_da"]
     if karabo_da[0] != "-1" and da not in karabo_da:
         continue
 
     instrument_source_name = detector.instrument_source(module)
     print('-', da, db_module, module, instrument_source_name)
 
     modules[da] = dict(
         db_module=db_module,
         module=module,
         raw_source_name=instrument_source_name,
         pdu_no=pdu_no,
     )
 
 constants = {}
 step_timer = StepTimer()
 ```
 
 %% Cell type:markdown id: tags:
 
 # Offset map
 
 %% Cell type:code id: tags:
 
 ``` python
-dark_run = runs[0]
+dark_run = run_high
+dark_creation_time = get_dir_creation_date(in_folder, dark_run)
+print(f"Using {dark_creation_time} as creation time of Offset constant.")
+
 for da, meta in modules.items():
     source_name = detector.instrument_source(meta["module"])
     image_key = detector.image_key
 
     display(Markdown(f"## {source_name}"))
 
     # read
     step_timer.start()
     file_da, _, _ = da.partition('/')
     dark_dc = RunDirectory(f"{in_folder}/r{dark_run:04d}",
                            include=f"RAW-R{dark_run:04d}-{file_da}-S*.h5")
     dark_dc = dark_dc.select([(source_name, image_key)])
     key_data = dark_dc[source_name, image_key]
 
     images_dark = key_data.ndarray()
     ntrain, npulse, ny, nx = images_dark.shape
 
     print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})")
     print(f"Image size: {ny} x {nx} px")
     step_timer.done_step("Read dark images")
 
     # process
     step_timer.start()
     dark = dffc.process_dark(images_dark)
 
     # put results in the dict
     conditions = detector.conditions(dark_dc, meta["module"])
     module_constants = constants.setdefault(meta["db_module"], {})
     module_constants["Offset"] = dict(
-        conditions=conditions, data=dark, pdu_no=meta["pdu_no"])
+        conditions=conditions, data=dark, pdu_no=meta["pdu_no"],
+        creation_time=dark_creation_time
+    )
     step_timer.done_step("Process dark images")
     display()
 
     # draw plots
     step_timer.start()
     plot_camera_image(dark)
     plt.show()
     step_timer.done_step("Draw offsets")
 ```
 
 %% Cell type:markdown id: tags:
 
 # Flat-field PCA decomposition
 
 %% Cell type:code id: tags:
 
 ``` python
-flat_run = runs[1]
+flat_run = run_low
+flat_creation_time = get_dir_creation_date(in_folder, flat_run)
+print(f"Using {flat_creation_time} as creation time of DynamicFF constant.")
+
 for da, meta in modules.items():
     source_name = detector.instrument_source(meta["module"])
     image_key = detector.image_key
 
     display(Markdown(f"## {source_name}"))
 
     # read
     step_timer.start()
     file_da, _, _ = da.partition('/')
     flat_dc = RunDirectory(f"{in_folder}/r{flat_run:04d}",
                            include=f"RAW-R{flat_run:04d}-{file_da}-S*.h5")
     flat_dc = flat_dc.select([(source_name, image_key)])
     key_data = flat_dc[source_name][image_key]
 
     images_flat = key_data.ndarray()
     ntrain, npulse, ny, nx = images_flat.shape
 
     print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})")
     print(f"Image size: {ny} x {nx} px")
     step_timer.done_step("Read flat-field images")
 
     # process
     step_timer.start()
     flat, components, explained_variance_ratio = dffc.process_flat(
         images_flat, dark, n_components)
     flat_data = np.concatenate([flat[None, ...], components])
 
     # put results in the dict
     conditions = detector.conditions(flat_dc, meta["module"])
     module_constants = constants.setdefault(meta["db_module"], {})
     module_constants["DynamicFF"] = dict(
-        conditions=conditions, data=flat_data, pdu_no=meta["pdu_no"]
+        conditions=conditions, data=flat_data, pdu_no=meta["pdu_no"],
+        creation_time=flat_creation_time
     )
     step_timer.done_step("Process flat-field images")
 
     # draw plots
     step_timer.start()
     display(Markdown("### Average flat-field"))
     plot_camera_image(flat)
     plt.show()
 
     display(Markdown("### Explained variance ratio"))
     fig, ax = plt.subplots(1, 1, figsize=(10,4), tight_layout=True)
     ax.semilogy(explained_variance_ratio, 'o')
     ax.set_xticks(np.arange(len(explained_variance_ratio)))
     ax.set_xlabel("Component no.")
     ax.set_ylabel("Variance fraction")
     plt.show()
 
     display(Markdown("### The first principal components (up to 20)"))
     plot_images(components[:20], figsize=(13, 8))
     plt.show()
 
     step_timer.done_step("Draw flat-field")
 ```
 
 %% Cell type:markdown id: tags:
 
 ## Calibration constants
 
 %% Cell type:code id: tags:
 
 ``` python
 step_timer.start()
 
 # Output Folder Creation:
 os.makedirs(out_folder, exist_ok=True)
 
-def inject_ccv(metadata_folder, calibration, cond, pdu, proposal, runs, const_file, begin_at):
-    print("* Send to db:", const_file)
-    print("  -", metadata_folder)
+def inject_ccv(in_folder, metadata_folder, runs, calibration, cond, pdu, const_input, begin_at):
+    print("* Send to db:", const_input)
+    print("  - in folder:", in_folder)
+    print("  - metadata folder:", metadata_folder)
+    print("  - runs:", runs)
     print("  -", calibration)
     print("  -", cond)
-    print("  - proposal", proposal)
-    print("  - runs", runs)
     print("  -", begin_at)
 
 for db_module, module_constants in constants.items():
     for constant_name, constant in module_constants.items():
         conditions = constant["conditions"]
         conditions_dict = conditions.make_dict(
             conditions.calibration_types[constant_name])
 
         data_to_store = {db_module: {constant_name: {'0': {
             'conditions': conditions_dict,
             'data': constant["data"],
         }}}}
 
         ofile = f"{out_folder}/const_{constant_name}_{db_module}.h5"
         if os.path.isfile(ofile):
             print(f'File {ofile} already exists and will be overwritten')
 
         save_dict_to_hdf5(data_to_store, ofile)
         if db_output:
             inject_ccv(
-                metadata_folder, constant_name, conditions,
-                pdus["data"][constant["pdu_no"]],
-                proposal, runs, ofile, creation_time
+                in_folder, metadata_folder, [dark_run, flat_run],
+                constant_name, conditions, pdus["data"][constant["pdu_no"]],
+                ofile, constant["creation_time"]
             )
 
         if not local_output:
             os.unlink(ofile)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 print(f"Total processing time {step_timer.timespan():.01f} s")
 step_timer.print_summary()
 ```