[AGIPD][CORRECT] Use calcat_interface and remove precorrection notebook

Description

Updating AGIPD correction notebooks for using calcat_interface and avoiding calibrationDBRemote and cal_db_interactive

Note: This MR is based on https://git.xfel.eu/detectors/pycalibration/-/merge_requests/773

Related to https://git.xfel.eu/calibration/planning/-/issues/150 and https://git.xfel.eu/calibration/planning/-/issues/168

1. Remove notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
2. Remove cal_tools/agipdlib functions for loading constants from YAML or DB. Also, remove related tests.
3. Move to retrieve and initialize constants to the AGIPD correction notebook.
4. Use a new feature for displaying the markdown table for retrieved constants.
5. Fix a bug when displaying one retrieved constant in CalibrationData metadata object.
6. Use a fragment file to store retrieved constants.
7. Clean-ups around related code.

How Has This Been Tested?

• Testing..

Relevant Documents (optional)

Types of changes

• New feature (non-breaking change which adds functionality)
• Bug fix (non-breaking change which fixes an issue)

Checklist:

Reviewers

@schmidtp @kluyvert

tests/test_cal_tools.py

@@ -9,6 +9,7 @@ from extra_data import open_run
 from iCalibrationDB import Conditions, ConstantMetaData, Constants
 
 from cal_tools.agipdlib import AgipdCorrections, CellRange
+from cal_tools.calcat_interface import CalCatError
 from cal_tools.plotting import show_processed_modules
 from cal_tools.tools import (
     creation_date_file_metadata,
@@ -25,6 +26,7 @@ from cal_tools.tools import (
 ACQ_RATE = 1.1
 BIAS_VOLTAGE = 300
 GAIN_SETTING = 0
+GAIN_MODE = 0
 INTEGRATION_TIME = 12
 MEM_CELLS = 352
 PHOTON_ENERGY = 9.2
@@ -405,7 +407,7 @@ def test_get_pdu_from_db(_agipd_const_cond):
 # TODO add a marker for accessing zmq end_point
 @pytest.mark.requires_gpfs
 @pytest.mark.requires_caldb
-def test_initialize_from_db():
+def test_raise_initialize_from_db():
     creation_time = datetime.strptime(
         "2020-01-07 13:26:48.00", "%Y-%m-%d %H:%M:%S.%f")
 
@@ -417,44 +419,55 @@ def test_initialize_from_db():
         modules=[0],
         constant_shape=(3, MEM_CELLS, 512, 128))
 
+    # CalCatError: calibration_constant_version not found !
+    with pytest.raises(CalCatError):
+        agipd_corr.initialize_from_db(
+            karabo_id="TEST_DET_CAL_CI-1",
+            karabo_da="TEST_DET_CAL_DA1",
+            creation_time=creation_time,
+            memory_cells=MEM_CELLS,
+            bias_voltage=BIAS_VOLTAGE,
+            photon_energy=PHOTON_ENERGY,
+            gain_setting=GAIN_SETTING,
+            gain_mode=GAIN_MODE,
+            acquisition_rate=ACQ_RATE,
+            integration_time=INTEGRATION_TIME,
+            module_idx=0,
+        )
+
+# TODO add a marker for accessing zmq end_point
+@pytest.mark.requires_gpfs
+@pytest.mark.requires_caldb
+def test_initialize_from_db():
+    creation_time = datetime.strptime(
+        "2020-01-07 13:26:48.00", "%Y-%m-%d %H:%M:%S.%f")
+
+    agipd_corr = AgipdCorrections(
+        max_cells=MEM_CELLS,
+        cell_sel=CellRange([0, 500, 1], MEM_CELLS))
+
+    agipd_corr.allocate_constants(
+        modules=[0],
+        constant_shape=(3, MEM_CELLS, 512, 128))
+    agipd_corr
     dark_const_time_dict = agipd_corr.initialize_from_db(
-        karabo_id="TEST_DET_CAL_CI-1",
-        karabo_da="TEST_DET_CAL_DA1",
-        cal_db_interface=CAL_DB_INTERFACE,
+        karabo_id=AGIPD_KARABO_ID,
+        karabo_da="AGIPD00",
         creation_time=creation_time,
         memory_cells=MEM_CELLS,
         bias_voltage=BIAS_VOLTAGE,
         photon_energy=PHOTON_ENERGY,
         gain_setting=GAIN_SETTING,
-        acquisition_rate=ACQ_RATE,
+        gain_mode=GAIN_MODE,
         integration_time=INTEGRATION_TIME,
+        acquisition_rate=ACQ_RATE,
         module_idx=0,
-        only_dark=False,
-    )
-
-    assert dark_const_time_dict == {
-        "Offset": None,
-        "Noise": None,
-        "ThresholdsDark": None,
-        "BadPixelsDark": None,
-    }
-
-    dark_const_time_dict = agipd_corr.initialize_from_db(
-        karabo_id=AGIPD_KARABO_ID,
-        karabo_da="AGIPD00",
-        cal_db_interface=CAL_DB_INTERFACE,
-        creation_time=creation_time,
-        memory_cells=MEM_CELLS, bias_voltage=BIAS_VOLTAGE,
-        photon_energy=PHOTON_ENERGY, gain_setting=GAIN_SETTING,
-        integration_time=INTEGRATION_TIME,
-        acquisition_rate=ACQ_RATE, module_idx=0,
-        only_dark=False,
     )
 
     # A retrieved constant has a value of datetime creation_time
-    assert isinstance(dark_const_time_dict["Offset"], datetime)
-    assert list(dark_const_time_dict.keys()) == [
-        "Offset", "Noise", "ThresholdsDark", "BadPixelsDark"]
+    assert isinstance(dark_const_time_dict["Offset"], str)
+    assert set(dark_const_time_dict.keys()) == {
+        "Offset", "Noise", "ThresholdsDark", "BadPixelsDark"}
 
 
 def test_module_index_to_qm():