[JUNGFRAU][CORRECT] Using calcat interface

Start using calcat interface with Jungfrau correction.

Related to https://git.xfel.eu/calibration/planning/-/issues/150

Description

notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb:

• Retrieve all constants in one go and validate the availability for the constants and raise Error/Warning accordingly.
• Updating the YAML metadata keys based on the attributes in the CalibrationData metadata object.

notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb:

• Using calcat interface to retrieve constants if YAML is not available.

How Has This Been Tested?

Tested against the reference test runs.

Relevant Documents (optional)

Types of changes

• New feature (non-breaking change which adds functionality)

Checklist:

- [ ] update setup.py with extra_geom release of this change https://github.com/European-XFEL/EXtra-geom/pull/198 - [ ] update setup.py with extra_data release of this change https://github.com/European-XFEL/EXtra-data/pull/379

Reviewers

@kluyvert @schmidtp

notebooks/LPD/LPD_Correct_Fast.ipynb

@@ -26,7 +26,7 @@
     "metadata_folder = ''  # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n",
     "sequences = [-1]  # Sequences to correct, use [-1] for all\n",
     "modules = [-1]  # Modules indices to correct, use [-1] for all, only used when karabo_da is empty\n",
-    "karabo_da = ['all']  # Data aggregators names to correct, use [''] for all\n",
+    "karabo_da = ['']  # Data aggregators names to correct, use [''] for all\n",
     "run = 10  # run to process, required\n",
     "\n",
     "# Source parameters\n",
@@ -104,7 +104,10 @@
     "\n",
     "from extra_data.components import LPD1M\n",
     "\n",
+<<<<<<< HEAD
     "from cal_tools.calcat_interface import CalCatError, LPD_CalibrationData\n",
+=======
+>>>>>>> remove LPD changes
     "from cal_tools.lpdalgs import correct_lpd_frames\n",
     "from cal_tools.tools import CalibrationMetadata, calcat_creation_time\n",
     "from cal_tools.files import DataFile\n",
@@ -135,20 +138,16 @@
     "cal_db_root = Path(cal_db_root)\n",
     "\n",
     "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
-    "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
-    "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
     "\n",
     "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
     "print(f'Using {creation_time.isoformat()} as creation time')\n",
     "\n",
     "# Pick all modules/aggregators or those selected.\n",
-    "if karabo_da == ['all']:\n",
-    "    if modules == [-1]:\n",
+    "if not karabo_da or karabo_da == ['']:\n",
+    "    if not modules or modules == [-1]:\n",
     "        modules = list(range(16))\n",
     "\n",
     "    karabo_da = [f'LPD{i:02d}' for i in modules]\n",
-    "else:\n",
-    "    modules = [int(x[-2:]) for x in karabo_da]\n",
     "    \n",
     "# Pick all sequences or those selected.\n",
     "if not sequences or sequences == [-1]:\n",
@@ -198,17 +197,6 @@
     "# Obtain and prepare calibration constants"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
-    "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
-    "const_yaml = metadata.setdefault(\"retrieved-constants\", {})"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -232,11 +220,23 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
+   "outputs": [],
+   "source": [
+<<<<<<< HEAD
+=======
+    "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+    "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
+    "const_yaml = metadata.setdefault(\"retrieved-constants\", {})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
    "outputs": [],
    "source": [
+>>>>>>> remove LPD changes
     "const_data = {}\n",
     "const_load_mp = psh.ProcessContext(num_workers=24)\n",
     "\n",
@@ -248,6 +248,7 @@
     "\n",
     "            dtype = np.uint32 if calibration_name.startswith('BadPixels') else np.float32\n",
     "\n",
+<<<<<<< HEAD
 <<<<<<< HEAD
     "            const_data[(da, calibration_name)] = dict(\n",
     "                path=Path(ccv['path']),\n",
@@ -255,6 +256,40 @@
     "                data=const_load_mp.alloc(shape=(256, 256, mem_cells, 3), dtype=dtype)\n",
     "            )\n",
     "else:  # Retrieve constants from CALCAT.\n",
+=======
+    "            const_data[(da, calibration_name)] = dict(\n",
+    "                path=Path(ccv['file-path']),\n",
+    "                dataset=ccv['dataset-name'],\n",
+    "                data=const_load_mp.alloc(shape=(256, 256, mem_cells, 3), dtype=dtype)\n",
+    "            )\n",
+    "else:  # Retrieve constants from CALCAT.\n",
+    "    dark_calibrations = {\n",
+    "        1: 'Offset',  # np.float32\n",
+    "        14: 'BadPixelsDark'  # should be np.uint32, but is np.float64\n",
+    "    }\n",
+    "\n",
+    "    dark_condition = [\n",
+    "        dict(parameter_id=1, value=bias_voltage),  # Sensor bias voltage\n",
+    "        dict(parameter_id=7, value=mem_cells),  # Memory cells\n",
+    "        dict(parameter_id=15, value=capacitor),  # Feedback capacitor\n",
+    "        dict(parameter_id=13, value=256),  # Pixels X\n",
+    "        dict(parameter_id=14, value=256),  # Pixels Y\n",
+    "    ]\n",
+    "\n",
+    "    illuminated_calibrations = {\n",
+    "        20: 'BadPixelsFF',  # np.uint32\n",
+    "        42: 'GainAmpMap',  # np.float32\n",
+    "        43: 'FFMap',  # np.float32\n",
+    "        44: 'RelativeGain'  # np.float32\n",
+    "    }\n",
+    "\n",
+    "    illuminated_condition = dark_condition.copy()\n",
+    "    illuminated_condition += [\n",
+    "        dict(parameter_id=3, value=photon_energy),  # Source energy\n",
+    "        dict(parameter_id=25, value=category)  # category\n",
+    "    ]\n",
+    "\n",
+>>>>>>> remove LPD changes
     "    print('Querying calibration database', end='', flush=True)\n",
     "    start = perf_counter()\n",
     "    for calibrations, condition in [\n",
@@ -264,8 +299,12 @@
     "        resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n",
     "            client, karabo_id, list(calibrations.keys()),\n",
     "            {'parameters_conditions_attributes': condition},\n",
+<<<<<<< HEAD
     "            karabo_da='', event_at=creation_time.isoformat()\n",
     "        )\n",
+=======
+    "            karabo_da='', event_at=creation_time.isoformat(), snapshot_at=None)\n",
+>>>>>>> remove LPD changes
     "\n",
     "        if not resp['success']:\n",
     "            raise RuntimeError(resp)\n",
@@ -284,6 +323,7 @@
     "            )\n",
     "        print('.', end='', flush=True)\n",
     "            \n",
+<<<<<<< HEAD
 =======
     "else:\n",
     "    lpd_cal = LPD_CalibrationData(\n",
@@ -319,6 +359,30 @@
     "print(f'{total_time:.1f}s')"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def load_constant_dataset(wid, index, const_descr):\n",
+    "    ccv_entry = const_data[const_descr]\n",
+    "    \n",
+    "    with h5py.File(cal_db_root / ccv_entry['path'], 'r') as fp:\n",
+    "        fp[ccv_entry['dataset'] + '/data'].read_direct(ccv_entry['data'])\n",
+    "        \n",
+    "    print('.', end='', flush=True)\n",
+    "\n",
+    "print('Loading calibration data', end='', flush=True)\n",
+    "start = perf_counter()\n",
+    "const_load_mp.map(load_constant_dataset, list(const_data.keys()))\n",
+=======
+>>>>>>> remove LPD changes
+    "total_time = perf_counter() - start\n",
+    "\n",
+    "print(f'{total_time:.1f}s')"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -588,7 +652,8 @@
     "ExecuteTime": {
      "end_time": "2018-11-13T18:24:57.547563Z",
      "start_time": "2018-11-13T18:24:56.995005Z"
-    }
+    },
+    "scrolled": false
    },
    "outputs": [],
    "source": [