[AGIPD][CORRECT] Split FF and PC queries

Description

This is related to this ticket https://redmine.xfel.eu/issues/160639

Since using calcat_interface with AGIPD I was using SplitConditionCalibrationData to retrieve both PC and FF constants. PC constants are dark and FF constants are illuminated.

As the mentioned runs in the ticket were running in High CDS, there were no PC constants for the conditions but there were available FF. However, it was report that no FF nor PC constants were found because the whole function failed trying to query PC.

This fix is splitting the query for PC and FF to avoid such errors. The same logic is hotfixed in production at the moment.

How Has This Been Tested?

Relevant Documents (optional)

Types of changes

• Bug fix (non-breaking change which fixes an issue)

Checklist:

Reviewers

@schmidtp @kluyvert

notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb

@@ -557,6 +557,21 @@
     "## Retrieving constants"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_constants_and_update_metadata(cal_data, main_metadata, constants):\n",
+    "    try:\n",
+    "        metadata = cal_data.metadata(constants)\n",
+    "        for key, value in metadata.items():\n",
+    "            main_metadata.setdefault(key, {}).update(value)\n",
+    "    except CalCatError as e:  # TODO: replace when API errors are improved.\n",
+    "        warning(f\"CalCatError: {e}\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -584,24 +599,21 @@
     "dark_constants = [\"Offset\", \"Noise\", \"BadPixelsDark\"]\n",
     "if not gain_mode:  # Adaptive gain\n",
     "    dark_constants.append(\"ThresholdsDark\")\n",
-    "gain_constants = []\n",
+    "\n",
+    "agipd_metadata = agipd_cal.metadata(dark_constants)\n",
+    "\n",
+    "agipd_cal.gain_mode = None  # gain_mode is not used for gain constants\n",
+    "pc_constants, ff_constants = [], []\n",
     "if any(agipd_corr.pc_bools):\n",
-    "    gain_constants += [\"SlopesPC\", \"BadPixelsPC\"]\n",
+    "    pc_constants = [\"SlopesPC\", \"BadPixelsPC\"]\n",
+    "    get_constants_and_update_metadata(\n",
+    "        agipd_cal, agipd_metadata, pc_constants)\n",
+    "\n",
     "if agipd_corr.corr_bools.get('xray_corr'):\n",
-    "    gain_constants += agipd_cal.illuminated_calibrations\n",
+    "    ff_constants = agipd_cal.illuminated_calibrations\n",
+    "    get_constants_and_update_metadata(\n",
+    "        agipd_cal, agipd_metadata, ff_constants)\n",
     "\n",
-    "# First retrieve dark constants\n",
-    "agipd_metadata = agipd_cal.metadata(dark_constants)\n",
-    "if gain_constants:\n",
-    "    # Then retrieve gain constants without\n",
-    "    # using the `gain_mode` condition.\n",
-    "    agipd_cal.gain_mode = None\n",
-    "    try:\n",
-    "        illum_metadata = agipd_cal.metadata(gain_constants)\n",
-    "        for key, value in illum_metadata.items():\n",
-    "            agipd_metadata.setdefault(key, {}).update(value)\n",
-    "    except CalCatError as e:  # TODO: replace when API errors are improved.\n",
-    "        warning(f\"CalCatError: {e}\")\n",
     "step_timer.done_step(\"Constants were retrieved in\")\n",
     "\n",
     "print(\"Preparing constants (\"\n",
@@ -634,7 +646,7 @@
     "        karabo_da.drop(da)\n",
     "        modules.drop(mod)\n",
     "\n",
-    "    warn_missing_constants = set(dark_constants + gain_constants)\n",
+    "    warn_missing_constants = set(dark_constants + pc_constants + ff_constants)\n",
     "    warn_missing_constants -= error_missing_constants\n",
     "    warn_missing_constants -= set(calibrations)\n",
     "    if warn_missing_constants:\n",

[Egor Sobolev]

This returns a set. dark_constants, pc_constants are lists. In line 649 they are concatenated, that raises exception

[Egor Sobolev]

TypeError                                 Traceback (most recent call last)
<ipython-input-19-32ef92fb1091> in <module>
     15 modules.drop(mod)
     16
---> 17 warn_missing_constants = set(dark_constants + pc_constants + ff_constants)
     18 warn_missing_constants -= error_missing_constants
     19 warn_missing_constants -= set(calibrations)
TypeError: can only concatenate list (not "set") to list

[Karim Ahmed]

Thank you @esobolev for reporting this! I will fix this. This didn't fail in the tests because x-ray gain is not used in the reference test. Which is something I am changing !902 (merged)