[Generic][Shimadzu] Dynamic flat-field characterization and correction for MHz microscopy

This implements:

1. The characterization of dark and flat fields of Shimadzu HPVX2 images
2. The correction for Shimadzu HPVX2 images by applying offsets subtraction and dynamic flat-field correction.

Depends on the https://github.com/sarlotabirnsteinova/OnlineVisualization

This is needed for MHz microscopy at SPB

Features:

• Split on sequence files by 200 trains, ignoring original splitting
• Creates new source name SPB_EHD_MIC/CAM/HPVX2_1:daqOutput -> SPB_EHD_MIC/CORR/HPVX2_1:output

Corrected files structure

CORR-R0003-HPVX01-S00000.h5 ├INDEX │ ├SPB_EHD_MIC │ │ └CORR │ │ ├HPVX2_1:output │ │ │ └index_group │ │ │ ├count [uint64: 15] │ │ │ └first [uint64: 15] │ │ └HPVX2_2:output │ │ └index_group │ │ ├count [uint64: 15] │ │ └first [uint64: 15] │ ├flag [int32: 15] │ ├origin [int32: 15] │ ├timestamp [uint64: 15] │ └trainId [uint64: 15] ├INSTRUMENT │ └SPB_EHD_MIC │ └CORR │ ├HPVX2_1:output │ │ └data │ │ └image │ │ ├binning [uint64: 9 × 3] │ │ ├dimTypes [int32: 9 × 3] │ │ ├dims [uint64: 9 × 3] │ │ ├flipX [uint8: 9] │ │ ├flipY [uint8: 9] │ │ ├pixels [float32: 9 × 128 × 250 × 400] │ │ ├roiOffsets [uint64: 9 × 3] │ │ └rotation [int32: 9] │ └HPVX2_2:output │ └data │ └image │ ├binning [uint64: 10 × 3] │ ├dimTypes [int32: 10 × 3] │ ├dims [uint64: 10 × 3] │ ├flipX [uint8: 10] │ ├flipY [uint8: 10] │ ├pixels [float32: 10 × 128 × 250 × 400] │ ├roiOffsets [uint64: 10 × 3] │ └rotation [int32: 10] └METADATA ├creationDate [ASCII string: 1] ├daqLibrary [ASCII string: 1] ├dataFormatVersion [ASCII string: 1] ├dataSources │ ├dataSourceId [ASCII string: 2] │ ├deviceId [ASCII string: 2] │ └root [ASCII string: 2] ├karaboFramework [ASCII string: 1] ├proposalNumber [uint32: 1] ├runNumber [uint32: 1] ├sequenceNumber [uint32: 1] └updateDate [ASCII string: 1]

Reports:

• characterization report: dark_runs_1_2_v0.pdf
• correction report: r0003.pdf

cc @schmidtp

notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb

 in_folder = "/gpfs/exfel/exp/SPB/202430/p900425/raw"  # input folder, required
 out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu'  # output folder, required
 metadata_folder = ""  # Directory containing calibration_metadata.yml when run by xfel-calibrate
-runs = [1, 2]  # list of two run numbers: dark field and flat field
+run_high = 1 # run number in which dark data was recorded, required
+run_low = 2 # run number in which flat-field data was recorded, required
+operation_mode = "TI_DynamicFF"  # Detector operation mode, optional (defaults to "TI_DynamicFF")
 
 # Data files parameters.
 karabo_da = ['-1'] # data aggregators
 karabo_id = "SPB_MIC_HPVX2" # karabo prefix of Shimadzu HPV-X2 devices
 
-#receiver_id = "PNCCD_FMT-0" # inset for receiver devices
-#path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5'  # the template to use to access data
-#instrument_source_template = 'SPB_EHD_MIC/CAM/HPVX2_{module}:daqOutput'  # data source path in h5file.
-#instrument_source_template = 'SPB_EHD_HPVX2_{module}/CAM/CAMERA:daqOutput'
-#image_key = "data.image.pixels"  # image data key in Karabo or exdf notation
-
-#db_module_template = "Shimadzu_HPVX2_{}"
-
 # Database access parameters.
-use_dir_creation_date = True  # use dir creation date as data production reference date
 cal_db_interface = "tcp://max-exfl-cal001:8021"  # calibration DB interface to use
-cal_db_timeout = 300000 # timeout on caldb requests
 db_output = True # if True, the notebook sends dark constants to the calibration database
 local_output = True # if True, the notebook saves dark constants locally
-creation_time = "" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. 2019-07-04 11:02:41.00
 
-n_components = 50  # Number of principal components to compute
+# Calibration constants parameters
+n_components = 50  # Number of principal components of flat-field to compute (default: 50)
 %% Cell type:code id: tags:
 from cal_tools.step_timing import StepTimer
 from cal_tools.tools import (
     get_dir_creation_date,
-#    get_pdu_from_db,
     get_random_db_interface,
     get_report,
-#    save_const_to_h5,
     save_dict_to_hdf5,
-#    send_to_db,
     run_prop_seq_from_path,
 )
 from cal_tools.restful_config import calibration_client
 ``` python
-creation_time=None
-if use_dir_creation_date:
-    creation_time = get_dir_creation_date(in_folder, max(runs))
-
-print(f"Using {creation_time} as creation time of constant.")
-
-run, proposal, seq = run_prop_seq_from_path(in_folder)
-#file_loc = f'proposal: {prop}, runs: {dark_run} {flat_run}'
-
-# Read report path and create file location tuple to add with the injection
-#file_loc = f"proposal:{proposal} runs:" + ' '.join(str(run) for run in runs)
-
-report = get_report(metadata_folder)
 cal_db_interface = get_random_db_interface(cal_db_interface)
 print(f'Calibration database interface: {cal_db_interface}')
 print()
 ``` python
-dark_run = runs[0]
+dark_run = run_high
+dark_creation_time = get_dir_creation_date(in_folder, dark_run)
+print(f"Using {dark_creation_time} as creation time of Offset constant.")
+
 for da, meta in modules.items():
     source_name = detector.instrument_source(meta["module"])
     image_key = detector.image_key
     conditions = detector.conditions(dark_dc, meta["module"])
     module_constants = constants.setdefault(meta["db_module"], {})
     module_constants["Offset"] = dict(
-        conditions=conditions, data=dark, pdu_no=meta["pdu_no"])
+        conditions=conditions, data=dark, pdu_no=meta["pdu_no"],
+        creation_time=dark_creation_time
+    )
     step_timer.done_step("Process dark images")
     display()
 
 ``` python
-flat_run = runs[1]
+flat_run = run_low
+flat_creation_time = get_dir_creation_date(in_folder, flat_run)
+print(f"Using {flat_creation_time} as creation time of DynamicFF constant.")
+
 for da, meta in modules.items():
     source_name = detector.instrument_source(meta["module"])
     image_key = detector.image_key
     conditions = detector.conditions(flat_dc, meta["module"])
     module_constants = constants.setdefault(meta["db_module"], {})
     module_constants["DynamicFF"] = dict(
-        conditions=conditions, data=flat_data, pdu_no=meta["pdu_no"]
+        conditions=conditions, data=flat_data, pdu_no=meta["pdu_no"],
+        creation_time=flat_creation_time
     )
     step_timer.done_step("Process flat-field images")
 
 # Output Folder Creation:
 os.makedirs(out_folder, exist_ok=True)
 
-def inject_ccv(metadata_folder, calibration, cond, pdu, proposal, runs, const_file, begin_at):
-    print("* Send to db:", const_file)
-    print("  -", metadata_folder)
+def inject_ccv(in_folder, metadata_folder, runs, calibration, cond, pdu, const_input, begin_at):
+    print("* Send to db:", const_input)
+    print("  - in folder:", in_folder)
+    print("  - metadata folder:", metadata_folder)
+    print("  - runs:", runs)
     print("  -", calibration)
     print("  -", cond)
-    print("  - proposal", proposal)
-    print("  - runs", runs)
     print("  -", begin_at)
 
 for db_module, module_constants in constants.items():
         save_dict_to_hdf5(data_to_store, ofile)
         if db_output:
             inject_ccv(
-                metadata_folder, constant_name, conditions,
-                pdus["data"][constant["pdu_no"]],
-                proposal, runs, ofile, creation_time
+                in_folder, metadata_folder, [dark_run, flat_run],
+                constant_name, conditions, pdus["data"][constant["pdu_no"]],
+                ofile, constant["creation_time"]
             )
 
         if not local_output: