From 58845f6fd7bb728fb0687e31ac078658551bd911 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Wed, 14 Dec 2022 11:12:44 +0100
Subject: [PATCH] remove Gotthard2 changes

---
 .../Gotthard2/Correction_Gotthard2_NBC.ipynb  | 188 ++++++++----------
 ...retrieve_constants_precorrection_NBC.ipynb | 159 ++++++++-------
 2 files changed, 165 insertions(+), 182 deletions(-)

diff --git a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
index 7bf1e4136..fc8913230 100644
--- a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
@@ -36,9 +36,10 @@
     "karabo_id_control = \"\"  # Control karabo ID. Set to empty string to use the karabo-id\n",
     "\n",
     "# Parameters for calibration database.\n",
+    "use_dir_creation_date = True  # use the creation data of the input dir for database queries.\n",
     "cal_db_interface = \"tcp://max-exfl016:8016#8025\"  # the database interface to use.\n",
     "cal_db_timeout = 180000  # timeout on caldb requests.\n",
-    "creation_time = \"\"  # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
+    "overwrite_creation_time = \"\"  # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n",
     "\n",
     "# Parameters affecting corrected data.\n",
     "constants_file = \"\"  # Use constants in given constant file path. /gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\n",
@@ -69,8 +70,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import datetime\n",
     "import warnings\n",
-    "from logging import warning\n",
+    "from functools import partial\n",
     "\n",
     "import h5py\n",
     "import pasha as psh\n",
@@ -80,16 +82,16 @@
     "from extra_data import RunDirectory, H5File\n",
     "from pathlib import Path\n",
     "\n",
-    "from calibration_client import CalibrationClient\n",
     "from cal_tools import h5_copy_except\n",
-    "from cal_tools.calcat_interface import CalCatError, GOTTHARD2_CalibrationData\n",
     "from cal_tools.gotthard2 import gotthard2algs, gotthard2lib\n",
-    "from cal_tools.restful_config import restful_config\n",
     "from cal_tools.step_timing import StepTimer\n",
     "from cal_tools.tools import (\n",
-    "    calcat_creation_time,\n",
+    "    get_constant_from_db_and_time,\n",
+    "    get_dir_creation_date,\n",
+    "    get_pdu_from_db,\n",
     "    CalibrationMetadata,\n",
     ")\n",
+    "from iCalibrationDB import Conditions, Constants\n",
     "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
     "\n",
     "warnings.filterwarnings('ignore')\n",
@@ -121,9 +123,14 @@
     "\n",
     "print(f\"Process modules: {karabo_da} for run {run}\")\n",
     "\n",
-    "# Run's creation time:\n",
-    "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
-    "print(f\"Creation time: {creation_time}\")"
+    "creation_time = None\n",
+    "if overwrite_creation_time:\n",
+    "    creation_time = datetime.datetime.strptime(\n",
+    "        overwrite_creation_time, \"%Y-%m-%d %H:%M:%S.%f\"\n",
+    "    )\n",
+    "elif use_dir_creation_date:\n",
+    "    creation_time = get_dir_creation_date(in_folder, run)\n",
+    "    print(f\"Using {creation_time} as creation time\")"
    ]
   },
   {
@@ -193,27 +200,6 @@
     "print(\"Single Photon:\", single_photon)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "b3613034",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Connect to CalCat.\n",
-    "calcat_config = restful_config['calcat']\n",
-    "client = CalibrationClient(\n",
-    "    base_api_url=calcat_config['base-api-url'],\n",
-    "    use_oauth2=calcat_config['use-oauth2'],\n",
-    "    client_id=calcat_config['user-id'],\n",
-    "    client_secret=calcat_config['user-secret'],\n",
-    "    user_email=calcat_config['user-email'],\n",
-    "    token_url=calcat_config['token-url'],\n",
-    "    refresh_url=calcat_config['refresh-url'],\n",
-    "    auth_url=calcat_config['auth-url'],\n",
-    "    scope='')"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "8c852392-bb19-4c40-b2ce-3b787538a92d",
@@ -225,33 +211,12 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "1817d0d8",
+   "id": "5717d722",
    "metadata": {},
    "outputs": [],
    "source": [
-    "da_to_pdu = {}\n",
     "# Used for old FXE (p003225) runs before adding Gotthard2 to CALCAT\n",
     "const_data = dict()\n",
-    "\n",
-    "g2_cal = GOTTHARD2_CalibrationData(\n",
-    "    detector_name=karabo_id,\n",
-    "    sensor_bias_voltage=bias_voltage,\n",
-    "    exposure_time=exposure_time,\n",
-    "    exposure_period=exposure_period,\n",
-    "    acquisition_rate=acquisition_rate,\n",
-    "    single_photon=single_photon,\n",
-    "    event_at=creation_time,\n",
-    "    client=client,\n",
-    ")\n",
-    "# Keep as long as it is essential to correct\n",
-    "# RAW data (FXE p003225) before the data mapping was added to CALCAT.\n",
-    "try:  # in case local constants are used with old RAW data. This can be removed in the future.\n",
-    "    for mod_info in g2_cal.physical_detector_units.values():\n",
-    "        da_to_pdu[mod_info[\"karabo_da\"]] = mod_info[\"physical_name\"]\n",
-    "except CalCatError as e:\n",
-    "    print(e)\n",
-    "    db_modules = [None] * len(karabo_da)\n",
-    "\n",
     "if constants_file:\n",
     "    for mod in karabo_da:\n",
     "        const_data[mod] = dict()\n",
@@ -260,59 +225,80 @@
     "            const_data[mod][\"LUT\"] = cfile[\"LUT\"][()]\n",
     "            const_data[mod][\"Offset\"] = cfile[\"offset_map\"][()].astype(np.float32)\n",
     "            const_data[mod][\"RelativeGain\"] = cfile[\"gain_map\"][()].astype(np.float32)\n",
-    "            const_data[mod][\"Mask\"] = cfile[\"bpix_ff\"][()].astype(np.uint32)\n",
-    "else:\n",
-    "    if const_yaml:\n",
-    "        const_data = dict()\n",
-    "        for mod in karabo_da:\n",
-    "            const_data[mod] = dict()\n",
+    "            const_data[mod][\"Mask\"] = cfile[\"bpix_ff\"][()].astype(np.uint32)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1cdbe818",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Conditions iCalibrationDB object.\n",
+    "condition = Conditions.Dark.Gotthard2(\n",
+    "    bias_voltage=bias_voltage,\n",
+    "    exposure_time=exposure_time,\n",
+    "    exposure_period=exposure_period,\n",
+    "    single_photon=single_photon,\n",
+    "    acquisition_rate=acquisition_rate,\n",
+    ")\n",
+    "\n",
+    "# TODO: Maybe this condition and previous cell can be removed later after the initial phase.\n",
+    "if not constants_file:\n",
+    "    # Prepare a dictionary of empty constants to loop on\n",
+    "    # it's keys and initiate non-retrieved constants.\n",
+    "    empty_lut = (np.arange(2 ** 12).astype(np.float64) * 2 ** 10 / 2 ** 12).astype(\n",
+    "        np.uint16\n",
+    "    )\n",
+    "    empty_lut = np.stack(1280 * [np.stack([empty_lut] * 2)], axis=0)\n",
+    "    empty_constants = {\n",
+    "        \"LUT\": empty_lut,\n",
+    "        \"Offset\": np.zeros((1280, 2, 3), dtype=np.float32),\n",
+    "        \"BadPixelsDark\": np.zeros((1280, 2, 3), dtype=np.uint32),\n",
+    "        \"RelativeGain\": np.ones((1280, 2, 3), dtype=np.float32),\n",
+    "        \"BadPixelsFF\": np.zeros((1280, 2, 3), dtype=np.uint32),\n",
+    "    }\n",
+    "\n",
+    "    for mod in karabo_da:\n",
+    "        const_data[mod] = dict()\n",
+    "        # Only used for printing timestamps within the loop.\n",
+    "        when = dict()\n",
+    "        # Check YAML file for constant metadata of file path and creation-time\n",
+    "        if const_yaml:\n",
     "            for cname, mdata in const_yaml[mod][\"constants\"].items():\n",
     "                const_data[mod][cname] = dict()\n",
-    "                if mdata[\"creation-time\"]:\n",
-    "                    with h5py.File(mdata[\"path\"], \"r\") as cf:\n",
+    "                when[cname] = mdata[\"creation-time\"]\n",
+    "                if when[cname]:\n",
+    "                    with h5py.File(mdata[\"file-path\"], \"r\") as cf:\n",
     "                        const_data[mod][cname] = np.copy(\n",
-    "                            cf[f\"{mdata['dataset']}/data\"])\n",
-    "    else:\n",
-    "        g2_cal = GOTTHARD2_CalibrationData(\n",
-    "            detector_name=karabo_id,\n",
-    "            sensor_bias_voltage=bias_voltage,\n",
-    "            exposure_time=exposure_time,\n",
-    "            exposure_period=exposure_period,\n",
-    "            acquisition_rate=acquisition_rate,\n",
-    "            single_photon=single_photon,\n",
-    "            event_at=creation_time,\n",
-    "            snapshot_at=creation_time,\n",
-    "        )\n",
-    "\n",
-    "        mdata_dict = {\"constants\": dict()}\n",
-    "\n",
-    "        constant_names = [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"]\n",
-    "        if gain_correction:\n",
-    "            constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n",
-    "\n",
-    "        # Retrieve metadata for all pnccd constants.\n",
-    "\n",
-    "        g2_metadata = g2_cal.ndarray_map(constant_names)\n",
-    "\n",
-    "    # Validate the constants availability and raise/warn correspondingly.\n",
-    "    for mod, calibrations in const_data.items():\n",
-    "        missing_dark_constants = set(\n",
-    "            c for c in [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"] if c not in calibrations.keys())\n",
-    "        missing_gain_constants = set(\n",
-    "            c for c in [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"] if gain_correction and c not in calibrations.keys())  # noqa\n",
-    "        if missing_dark_constants:\n",
-    "            raise KeyError(\n",
-    "                f\"Dark constants {missing_dark_constants} are not available for correction. Mod: {mod}\")\n",
-    "\n",
-    "        if gain_correction and missing_gain_constants:\n",
-    "            warning(f\"Gain constants {missing_gain_constants} are not retrieved. Mod {mod}\")\n",
-    "            gain_correction = False\n",
-    "\n",
-    "        # Create the mask array.\n",
+    "                            cf[f\"{mdata['dataset-name']}/data\"]\n",
+    "                        )\n",
+    "                else:\n",
+    "                    const_data[mod][cname] = empty_constants[cname]\n",
+    "        else:  # Retrieve constants from CALCAT. Missing YAML file or running notebook interactively.\n",
+    "            for cname, cempty in empty_constants.items():\n",
+    "                const_data[mod][cname] = dict()\n",
+    "                const_data[mod][cname], when[cname] = get_constant_from_db_and_time(\n",
+    "                    karabo_id=karabo_id,\n",
+    "                    karabo_da=mod,\n",
+    "                    cal_db_interface=cal_db_interface,\n",
+    "                    creation_time=creation_time,\n",
+    "                    timeout=cal_db_timeout,\n",
+    "                    print_once=False,\n",
+    "                    condition=condition,\n",
+    "                    constant=getattr(Constants.Gotthard2, cname)(),\n",
+    "                    empty_constant=cempty,\n",
+    "                )\n",
     "        bpix = const_data[mod][\"BadPixelsDark\"]\n",
-    "        if const_data[mod].get(\"BadPixelsFF\") is not None:\n",
-    "            bpix |= const_data[mod][\"BadPixelsFF\"]\n",
-    "        const_data[mod][\"Mask\"] = bpix"
+    "        bpix |= const_data[mod][\"BadPixelsFF\"]\n",
+    "        const_data[mod][\"Mask\"] = bpix\n",
+    "\n",
+    "        # Print timestamps for the retrieved constants.\n",
+    "        print(f\"Constants for module {mod}:\")\n",
+    "        for cname, ctime in when.items():\n",
+    "            print(f\"  {cname} injected at {ctime}\")\n",
+    "        del when"
    ]
   },
   {
diff --git a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
index 6effefce9..18aa31302 100644
--- a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
+++ b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
@@ -32,12 +32,13 @@
     "karabo_id_control = \"\"  # Control karabo ID. Set to empty string to use the karabo-id\n",
     "\n",
     "# Parameters for calibration database.\n",
-    "cal_db_interface = \"tcp://max-exfl016:8017#8025\"  # the database interface to use.\n",
+    "use_dir_creation_date = True  # use the creation data of the input dir for database queries.\n",
+    "cal_db_interface = \"tcp://max-exfl017:8017#8025\"  # the database interface to use.\n",
     "cal_db_timeout = 180000  # timeout on caldb requests.\n",
-    "creation_time = \"\"  # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
+    "overwrite_creation_time = \"2022-06-28 13:00:00.00\"  # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n",
     "\n",
     "# Parameters affecting corrected data.\n",
-    "constants_file = \"\"  # /gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\"  # Retrieve constants from local.\n",
+    "constants_file = \"\"#/gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\"  # Retrieve constants from local.\n",
     "offset_correction = True  # apply offset correction. This can be disabled to only apply LUT or apply LUT and gain correction for non-linear differential results.\n",
     "gain_correction = True  # apply gain correction.\n",
     "\n",
@@ -45,8 +46,8 @@
     "bias_voltage = -1  # Detector bias voltage, set to -1 to use value in raw file.\n",
     "exposure_time = -1.  # Detector exposure time, set to -1 to use value in raw file.\n",
     "exposure_period = -1.  # Detector exposure period, set to -1 to use value in raw file.\n",
-    "acquisition_rate = -1.  # Detector acquisition rate (1.1/4.5), set to -1 to use value in raw file.\n",
-    "single_photon = -1  # Detector single photon mode (High/Low CDS), set to -1 to use value in raw file.\n",
+    "acquisition_rate = 1.1  # Detector acquisition rate (1.1/4.5), set to -1 to use value in raw file.\n",
+    "single_photon = 0  # Detector single photon mode (High/Low CDS), set to -1 to use value in raw file.\n",
     "\n",
     "if constants_file:\n",
     "    print(\"Skipping constant retrieval. Specified constants_file is used.\")\n",
@@ -61,20 +62,20 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from logging import warning\n",
+    "import datetime\n",
+    "from functools import partial\n",
     "\n",
+    "import multiprocessing\n",
     "from extra_data import RunDirectory\n",
     "from pathlib import Path\n",
     "\n",
-    "from calibration_client import CalibrationClient\n",
-    "from cal_tools.calcat_interface import GOTTHARD2_CalibrationData\n",
     "from cal_tools.gotthard2 import gotthard2lib\n",
-    "from cal_tools.restful_config import restful_config\n",
     "from cal_tools.tools import (\n",
-    "    calcat_creation_time,\n",
+    "    get_dir_creation_date,\n",
     "    get_from_db,\n",
     "    CalibrationMetadata,\n",
-    ")"
+    ")\n",
+    "from iCalibrationDB import Conditions, Constants"
    ]
   },
   {
@@ -88,8 +89,6 @@
     "out_folder = Path(out_folder)\n",
     "out_folder.mkdir(parents=True, exist_ok=True)\n",
     "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
-    "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n",
-    "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
     "\n",
     "if not karabo_id_control:\n",
     "    karabo_id_control = karabo_id\n",
@@ -99,9 +98,14 @@
     "\n",
     "print(f\"Retrieve constants for modules: {karabo_da} for run {run}\")\n",
     "\n",
-    "# Run's creation time:\n",
-    "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
-    "print(f\"Creation time: {creation_time}\")"
+    "creation_time = None\n",
+    "if overwrite_creation_time:\n",
+    "    creation_time = datetime.datetime.strptime(\n",
+    "        overwrite_creation_time, \"%Y-%m-%d %H:%M:%S.%f\"\n",
+    "    )\n",
+    "elif use_dir_creation_date:\n",
+    "    creation_time = get_dir_creation_date(in_folder, run)\n",
+    "    print(f\"Using {creation_time} as creation time\")"
    ]
   },
   {
@@ -138,73 +142,60 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Connect to CalCat.\n",
-    "calcat_config = restful_config['calcat']\n",
-    "client = CalibrationClient(\n",
-    "    base_api_url=calcat_config['base-api-url'],\n",
-    "    use_oauth2=calcat_config['use-oauth2'],\n",
-    "    client_id=calcat_config['user-id'],\n",
-    "    client_secret=calcat_config['user-secret'],\n",
-    "    user_email=calcat_config['user-email'],\n",
-    "    token_url=calcat_config['token-url'],\n",
-    "    refresh_url=calcat_config['refresh-url'],\n",
-    "    auth_url=calcat_config['auth-url'],\n",
-    "    scope='')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "g2_cal = GOTTHARD2_CalibrationData(\n",
-    "    detector_name=karabo_id,\n",
-    "    sensor_bias_voltage=bias_voltage,\n",
+    "condition = Conditions.Dark.Gotthard2(\n",
+    "    bias_voltage=bias_voltage,\n",
     "    exposure_time=exposure_time,\n",
     "    exposure_period=exposure_period,\n",
-    "    acquisition_rate=acquisition_rate,\n",
     "    single_photon=single_photon,\n",
-    "    event_at=creation_time,\n",
-    "    snapshot_at=creation_time,\n",
-    "    client=client,\n",
+    "    acquisition_rate=acquisition_rate,\n",
     ")\n",
     "\n",
-    "mdata_dict = {\"constants\": dict()}\n",
-    "\n",
-    "constant_names = [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"]\n",
-    "if gain_correction:\n",
-    "    constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n",
-    "\n",
-    "# Retrieve metadata for all pnccd constants.\n",
+    "def get_constants_for_module(mod: str):\n",
+    "    \"\"\"Get calibration constants for given module for Gotthard2.\"\"\"\n",
+    "    retrieval_function = partial(\n",
+    "        get_from_db,\n",
+    "        karabo_id=karabo_id,\n",
+    "        karabo_da=mod,\n",
+    "        cal_db_interface=cal_db_interface,\n",
+    "        creation_time=creation_time,\n",
+    "        timeout=cal_db_timeout,\n",
+    "        verbosity=1,\n",
+    "        meta_only=True,\n",
+    "        load_data=False,\n",
+    "        empty_constant=None\n",
+    "    )\n",
     "\n",
-    "g2_metadata = g2_cal.metadata(constant_names)\n",
-    "# Validate the constants availability and raise/warn correspondingly.\n",
-    "for mod, ccv_dict in g2_metadata.items():\n",
-    "    missing_dark_constants = set(\n",
-    "        c for c in [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"] if c not in ccv_dict.keys())\n",
-    "    missing_gain_constants = set(\n",
-    "        c for c in [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"] if gain_correction and c not in ccv_dict.keys())  # noqa\n",
-    "    if missing_dark_constants:\n",
-    "        raise KeyError(\n",
-    "            f\"Dark constants {missing_dark_constants} are not available for correction. Mod: {mod}\")\n",
-    "    \n",
-    "    if gain_correction and missing_gain_constants:\n",
-    "        warning(f\"Gain constants {missing_gain_constants} are not retrieved. Mod {mod}\")\n",
+    "    mdata_dict = dict()\n",
+    "    mdata_dict[\"constants\"] = dict()\n",
+    "    constants = [\n",
+    "        \"LUT\", \"Offset\", \"BadPixelsDark\",\n",
+    "        \"RelativeGain\", \"BadPixelsFF\",\n",
+    "    ]\n",
+    "    for cname in constants:\n",
+    "        mdata_dict[\"constants\"][cname] = dict()\n",
+    "        if not gain_correction and cname in [\"BadPixelsFF\", \"RelativeGain\"]:\n",
+    "            continue\n",
+    "        _, mdata = retrieval_function(\n",
+    "            condition=condition,\n",
+    "            constant=getattr(Constants.Gotthard2, cname)(),\n",
+    "        )\n",
+    "        mdata_const = mdata.calibration_constant_version\n",
+    "        const_mdata = mdata_dict[\"constants\"][cname]\n",
+    "        # check if constant was successfully retrieved.\n",
+    "        if mdata.comm_db_success:\n",
+    "            const_mdata[\"file-path\"] = (\n",
+    "                f\"{mdata_const.hdf5path}\" f\"{mdata_const.filename}\"\n",
+    "            )\n",
+    "            const_mdata[\"dataset-name\"] = mdata_const.h5path\n",
+    "            const_mdata[\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+    "            mdata_dict[\"physical-detector-unit\"] = mdata_const.device_name\n",
+    "        else:\n",
+    "            const_mdata[\"file-path\"] = None\n",
+    "            const_mdata[\"creation-time\"] = None\n",
+    "    return mdata_dict, mod\n",
     "\n",
-    "# Add constants metadata in retrieved_constants dict.\n",
-    "for mod, ccv_dict in g2_metadata.items():\n",
-    "    mod_dict = retrieved_constants.setdefault(mod, dict())\n",
-    "    const_dict = mod_dict.setdefault(\"constants\", dict())\n",
-    "    for cname, ccv_metadata in ccv_dict.items():\n",
-    "        const_dict[cname] = {\n",
-    "                \"path\": str(g2_cal.caldb_root / ccv_metadata[\"path\"]),\n",
-    "                \"dataset\": ccv_metadata[\"dataset\"],\n",
-    "                \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
-    "            }\n",
-    "    mod_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
-    "    \n",
-    "print(f\"Stored retrieved constants in {metadata.filename}\")"
+    "with multiprocessing.Pool() as pool:\n",
+    "    results = pool.map(get_constants_for_module, karabo_da)"
    ]
   },
   {
@@ -213,20 +204,26 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n",
+    "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
     "timestamps = dict()\n",
     "\n",
-    "for mod in karabo_da:\n",
+    "for md_dict, mod in results:\n",
+    "    if mod in retrieved_constants:\n",
+    "        print(f\"Constant for {mod} already in calibration_metadata.yml, won't query again.\")  # noqa\n",
+    "        continue\n",
+    "    retrieved_constants[mod] = md_dict\n",
     "    module_timestamps = timestamps[mod] = dict()\n",
-    "    module_constants = retrieved_constants[mod]\n",
+    "\n",
     "    print(f\"Module: {mod}:\")\n",
-    "    for cname, mdata in module_constants[\"constants\"].items():\n",
+    "    for cname, mdata in md_dict[\"constants\"].items():\n",
     "        if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
     "            mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
     "        print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
     "\n",
     "    for cname in [\"Offset\", \"BadPixelsDark\", \"RelativeGain\", \"BadPixelsFF\"]:\n",
-    "        if cname in module_constants[\"constants\"]:\n",
-    "            module_timestamps[cname] = module_constants[\"constants\"][cname][\"creation-time\"]\n",
+    "        if cname in md_dict[\"constants\"]:\n",
+    "            module_timestamps[cname] = md_dict[\"constants\"][cname][\"creation-time\"]\n",
     "        else:\n",
     "            module_timestamps[cname] = \"NA\"\n",
     "\n",
-- 
GitLab