[GH2][CORRECT][DARK] Feat/add support for gh2 25um

Gotthard-II 25um Correction and Dark

Description

This Merge request is related to : https://git.xfel.eu/calibration/planning/-/issues/164

How Has This Been Tested?

• Tested against reference runs for GH2 50um.
• Tested against the production database for GH2 25um.

Relevant Documents (optional)

correct_gh2_25um.pdf

gh2_dark_25um.pdf

Correcting:

Into:

Types of changes

• New feature (non-breaking change which adds functionality)
• Bug fix (non-breaking change which fixes an issue)

Checklist:

• Follow up on learned lessons from LPDMini and propagate it here.
• Update the MR's history.
• Get Feedback on results from @mramilli

Reviewers

@schmidtp @hammerd

notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb

@@ -9,10 +9,10 @@
     "\n",
     "Author: European XFEL Detector Group, Version: 1.0\n",
     "\n",
-    "The following is a processing for offset, noise, and Badpixels maps using dark images taken with Gotthard2 detector.\n",
+    "The following is a processing for the dark constants (`Offset`, `Noise`, and `BadPixelsDark`) maps using dark images taken with Gotthard2 detector (GH2 50um or 25um).\n",
     "All constants are evaluated per strip, per pulse, and per memory cell. The maps are calculated for each gain stage that is acquired in 3 separate runs.\n",
     "\n",
-    "The three maps (calibration constants) can be injected to the database and stored locally."
+    "The three maps are of shape (stripes, cells, gains): (1280, 2, 3). They can be injected to the database (`db_output`) and/or stored locally (`local_output`)."
    ]
   },
   {
@@ -24,6 +24,7 @@
    "source": [
     "in_folder = \"/gpfs/exfel/exp/FXE/202231/p900298/raw\"  # the folder to read data from, required\n",
     "out_folder =  \"/gpfs/exfel/data/scratch/ahmedk/test/gotthard2/darks\"  # the folder to output to, required\n",
+    "metadata_folder = ''  # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
     "run_high = 7  # run number for G0 dark run, required\n",
     "run_med = 8  # run number for G1 dark run, required\n",
     "run_low = 9  # run number for G2 dark run, required\n",
@@ -33,15 +34,13 @@
     "karabo_da = [\"GH201\"]  # data aggregators\n",
     "receiver_template = \"RECEIVER\"  # receiver template used to read INSTRUMENT keys.\n",
     "control_template = \"CONTROL\"  # control template used to read CONTROL keys.\n",
-    "instrument_source_template = '{}/DET/{}:daqOutput'  # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'  # noqa\n",
     "ctrl_source_template = '{}/DET/{}'  # template for control source name (filled with karabo_id_control)\n",
     "karabo_id_control = \"\"  # Control karabo ID. Set to empty string to use the karabo-id\n",
     "\n",
     "# Parameters for the calibration database.\n",
-    "use_dir_creation_date = True\n",
     "cal_db_interface = \"tcp://max-exfl-cal001:8020\"  # calibration DB interface to use\n",
     "cal_db_timeout = 300000  # timeout on caldb requests\n",
-    "overwrite_creation_time = \"\"  # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n",
+    "creation_time = \"\"  # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
     "db_output = False  # Output constants to the calibration database\n",
     "local_output = True  # Output constants locally\n",
     "\n",
@@ -68,7 +67,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import datetime\n",
     "import numpy as np\n",
     "import matplotlib.pyplot as plt\n",
     "import pasha as psh\n",
@@ -76,17 +74,20 @@
     "from extra_data import RunDirectory\n",
     "from pathlib import Path\n",
     "\n",
+    "import yaml\n",
+    "from cal_tools.calcat_interface import CalCatApi\n",
     "from cal_tools.enums import BadPixels\n",
     "from cal_tools.gotthard2 import gotthard2algs, gotthard2lib\n",
     "from cal_tools.step_timing import StepTimer\n",
+    "from cal_tools.restful_config import calibration_client\n",
     "from cal_tools.tools import (\n",
-    "    get_dir_creation_date,\n",
+    "    calcat_creation_time,\n",
     "    get_constant_from_db_and_time,\n",
-    "    get_pdu_from_db,\n",
     "    get_report,\n",
     "    save_const_to_h5,\n",
     "    send_to_db,\n",
     ")\n",
+    "\n",
     "from iCalibrationDB import Conditions, Constants\n",
     "\n",
     "%matplotlib inline"
@@ -102,33 +103,34 @@
     "run_nums = [run_high, run_med, run_low]\n",
     "in_folder = Path(in_folder)\n",
     "out_folder = Path(out_folder)\n",
-    "out_folder.mkdir(exist_ok=True)\n",
-    "\n",
-    "print(f\"Process modules: {karabo_da}\")\n",
+    "out_folder.mkdir(parents=True, exist_ok=True)\n",
     "\n",
-    "run_dc = RunDirectory(in_folder / f\"r{run_high:04d}\")\n",
-    "file_loc = f\"proposal:{run_dc.run_metadata()['proposalNumber']} runs:{run_high} {run_med} {run_low}\"  # noqa\n",
-    "\n",
-    "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
     "ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)\n",
     "\n",
     "# Read report path to associate it later with injected constants.\n",
-    "report = get_report(out_folder)\n",
-    "\n",
-    "\n",
-    "if overwrite_creation_time:\n",
-    "    creation_time = datetime.datetime.strptime(\n",
-    "        overwrite_creation_time, \"%Y-%m-%d %H:%M:%S.%f\"\n",
-    "    )\n",
-    "elif use_dir_creation_date:\n",
-    "    creation_time = get_dir_creation_date(in_folder, run_high)\n",
-    "    print(f\"Using {creation_time.isoformat()} as creation time\")\n",
+    "report = get_report(metadata_folder)\n",
     "\n",
+    "# Run's creation time:\n",
+    "creation_time = calcat_creation_time(in_folder, run_high, creation_time)\n",
+    "print(f\"Creation time: {creation_time}\")\n",
     "\n",
     "if not karabo_id_control:\n",
     "    karabo_id_control = karabo_id"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c176a86f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "run_dc = RunDirectory(in_folder / f\"r{run_high:04d}\")\n",
+    "file_loc = f\"proposal:{run_dc.run_metadata()['proposalNumber']} runs:{run_high} {run_med} {run_low}\"  # noqa\n",
+    "\n",
+    "receivers = sorted(list(run_dc.select(f'{karabo_id}/DET/{receiver_template}*').all_sources))"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -193,7 +195,33 @@
     "single_photon = conditions[\"single_photon\"].pop()\n",
     "print(\"Single photon: \", single_photon)\n",
     "acquisition_rate = conditions[\"acquisition_rate\"].pop()\n",
-    "print(\"Acquisition rate: \", acquisition_rate)"
+    "print(\"Acquisition rate: \", acquisition_rate)\n",
+    "\n",
+    "gh2_detector = g2ctrl.get_det_type()\n",
+    "print(f\"Processing {gh2_detector} Gotthard2.\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f64bc150-cfcd-4f98-83f9-a982fdacedd7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "calcat = CalCatApi(client=calibration_client())\n",
+    "detector_id = calcat.detector(karabo_id)['id']\n",
+    "pdus_by_da = calcat.physical_detector_units(detector_id, pdu_snapshot_at=creation_time)\n",
+    "da_to_pdu = {da: p['physical_name'] for (da, p) in pdus_by_da.items()}\n",
+    "\n",
+    "if karabo_da != [\"\"]:\n",
+    "    # Filter DA connected to detector in CALCAT\n",
+    "    karabo_da = [da for da in karabo_da if da in da_to_pdu]\n",
+    "    # Exclude non selected DA from processing.\n",
+    "    da_to_pdu = {da: da_to_pdu[da] for da in karabo_da}\n",
+    "else:\n",
+    "    karabo_da = sorted(da_to_pdu.keys())\n",
+    "\n",
+    "print(f\"Processing {karabo_da}\")"
    ]
   },
   {
@@ -204,9 +232,7 @@
    "outputs": [],
    "source": [
     "def specify_trains_to_process(\n",
-    "    img_key_data: \"extra_data.KeyData\",  # noqa\n",
-    "    max_trains: int = 0,\n",
-    "    min_trains: int = 0,\n",
+    "    img_key_data: \"extra_data.KeyData\",\n",
     "):\n",
     "    \"\"\"Specify total number of trains to process.\n",
     "    Based on given min_trains and max_trains, if given.\n",
@@ -214,11 +240,11 @@
     "    Print number of trains to process and number of empty trains.\n",
     "    Raise ValueError if specified trains are less than min_trains.\n",
     "    \"\"\"\n",
-    "    # Specifies total number of trains to proccess.\n",
+    "    # Specifies total number of trains to process.\n",
     "    n_trains = img_key_data.shape[0]\n",
     "    all_trains = len(img_key_data.train_ids)\n",
     "    print(\n",
-    "        f\"{mod} has {all_trains - n_trains} \"\n",
+    "        f\"{receiver} has {all_trains - n_trains} \"\n",
     "        f\"trains with empty frames out of {all_trains} trains\"\n",
     "    )\n",
     "\n",
@@ -250,15 +276,6 @@
     "    exposure_period=exposure_period,\n",
     "    acquisition_rate=acquisition_rate,\n",
     "    single_photon=single_photon,\n",
-    ")\n",
-    "\n",
-    "db_modules = get_pdu_from_db(\n",
-    "    karabo_id=karabo_id,\n",
-    "    karabo_da=karabo_da,\n",
-    "    constant=Constants.Gotthard2.LUT(),\n",
-    "    condition=condition,\n",
-    "    cal_db_interface=cal_db_interface,\n",
-    "    snapshot_at=creation_time,\n",
     ")"
    ]
   },
@@ -272,7 +289,7 @@
     "def convert_train(wid, index, tid, d):\n",
     "    \"\"\"Convert a Gotthard2 train from 12bit to 10bit.\"\"\"\n",
     "    gotthard2algs.convert_to_10bit(\n",
-    "        d[instr_mod_src][\"data.adc\"], lut, data_10bit[index, ...]\n",
+    "        d[receiver][\"data.adc\"], lut, data_10bit[index, ...]\n",
     "    )"
    ]
   },
@@ -293,7 +310,7 @@
     "    np.uint16\n",
     ")\n",
     "empty_lut = np.stack(1280 * [np.stack([empty_lut] * 2)], axis=0)\n",
-    "for mod in karabo_da:\n",
+    "for mod, receiver in zip(karabo_da, receivers):\n",
     "\n",
     "    # Retrieve LUT constant\n",
     "    lut, time = get_constant_from_db_and_time(\n",
@@ -308,10 +325,7 @@
     "        print_once=False,\n",
     "    )\n",
     "    print(f\"Retrieved LUT constant with creation-time {time}\")\n",
-    "    # Path to pixels ADC values\n",
-    "    instr_mod_src = instrument_src.format(int(mod[-2:]))\n",
     "\n",
-    "    # TODO: Validate the final shape to store constants.\n",
     "    cshape = (1280, 2, 3)\n",
     "\n",
     "    offset_map[mod] = context.alloc(shape=cshape, dtype=np.float32)\n",
@@ -320,10 +334,10 @@
     "\n",
     "    for run_num, [gain, run_dc] in run_dcs_dict.items():\n",
     "        step_timer.start()\n",
-    "        n_trains = specify_trains_to_process(run_dc[instr_mod_src, \"data.adc\"])\n",
+    "        n_trains = specify_trains_to_process(run_dc[receiver, \"data.adc\"])\n",
     "\n",
     "        # Select requested number of trains to process.\n",
-    "        dc = run_dc.select(instr_mod_src, require_all=True).select_trains(\n",
+    "        dc = run_dc.select(receiver, require_all=True).select_trains(\n",
     "            np.s_[:n_trains]\n",
     "        )\n",
     "\n",
@@ -332,7 +346,7 @@
     "        step_timer.start()\n",
     "        # Convert 12bit data to 10bit\n",
     "        data_10bit = context.alloc(\n",
-    "            shape=dc[instr_mod_src, \"data.adc\"].shape, dtype=np.float32\n",
+    "            shape=dc[receiver, \"data.adc\"].shape, dtype=np.float32\n",
     "        )\n",
     "        context.map(convert_train, dc)\n",
     "        step_timer.done_step(\"convert to 10bit\")\n",
@@ -360,7 +374,7 @@
     "        context.map(offset_noise_cell, (even_data, odd_data))\n",
     "\n",
     "        # Split even and odd gain data.\n",
-    "        data_gain = dc[instr_mod_src, \"data.gain\"].ndarray()\n",
+    "        data_gain = dc[receiver, \"data.gain\"].ndarray()\n",
     "        even_gain = data_gain[:, 20::2, :]\n",
     "        odd_gain = data_gain[:, 21::2, :]\n",
     "        raw_g = 3 if gain == 2 else gain\n",
@@ -423,7 +437,7 @@
     "step_timer.start()\n",
     "g_name = [\"G0\", \"G1\", \"G2\"]\n",
     "\n",
-    "for mod, pdu in zip(karabo_da, db_modules):\n",
+    "for mod, pdu in da_to_pdu.items():\n",
     "    display(Markdown(f\"### Badpixels for module {mod}:\"))\n",
     "\n",
     "    badpixels_map[mod][\n",
@@ -436,19 +450,19 @@
     "    badpixels_map[mod][\n",
     "        ~np.isfinite(noise_map[mod])\n",
     "    ] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
-    "\n",
-    "    for cell in [0, 1]:\n",
-    "        fig, ax = plt.subplots(figsize=(10, 5))\n",
-    "        for g_idx in [0, 1, 2]:\n",
-    "            ax.plot(badpixels_map[mod][:, cell, g_idx], label=f\"G{g_idx} Bad pixel map\")\n",
-    "        ax.set_xticks(np.arange(0, 1281, 80))\n",
-    "        ax.set_xlabel(\"Stripes #\")\n",
-    "        ax.set_xlabel(\"BadPixels\")\n",
-    "        ax.set_title(f\"Cell {cell} - Module {mod} ({pdu})\")\n",
-    "        ax.set_ylim([0, 5])\n",
-    "        ax.legend()\n",
-    "        pass\n",
-    "step_timer.done_step(f\"Creating bad pixels constant and plotting it.\")"
+    "    if not local_output:\n",
+    "        for cell in [0, 1]:\n",
+    "            fig, ax = plt.subplots(figsize=(10, 5))\n",
+    "            for g_idx in [0, 1, 2]:\n",
+    "                ax.plot(badpixels_map[mod][:, cell, g_idx], label=f\"G{g_idx} Bad pixel map\")\n",
+    "            ax.set_xticks(np.arange(0, 1281, 80))\n",
+    "            ax.set_xlabel(\"Stripes #\")\n",
+    "            ax.set_xlabel(\"BadPixels\")\n",
+    "            ax.set_title(f\"BadPixels map - Cell {cell} - Module {mod} ({pdu})\")\n",
+    "            ax.set_ylim([0, 5])\n",
+    "            ax.legend()\n",
+    "            plt.show()\n",
+    "step_timer.done_step(f\"Creating bad pixels constant.\")"
    ]
   },
   {
@@ -458,18 +472,21 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "for mod, pdu in zip(karabo_da, db_modules):\n",
+    "if not local_output:\n",
     "    for cons, cname in zip([offset_map, noise_map], [\"Offset\", \"Noise\"]):\n",
-    "        for cell in [0, 1]:\n",
-    "            fig, ax = plt.subplots(figsize=(10, 5))\n",
-    "            for g_idx in [0, 1, 2]:\n",
-    "                ax.plot(cons[mod][:, cell, g_idx], label=f\"G{g_idx} {cname} map\")\n",
-    "\n",
-    "            ax.set_xlabel(\"Stripes #\")\n",
-    "            ax.set_xlabel(cname)\n",
-    "            ax.set_title(f\"{cname} map - Cell {cell} - Module {mod} ({pdu})\")\n",
-    "            ax.legend()\n",
-    "            pass"
+    "        for mod, pdu in da_to_pdu.items():\n",
+    "            display(Markdown(f\"### {cname} for module {mod}:\"))\n",
+    "\n",
+    "            for cell in [0, 1]:\n",
+    "                fig, ax = plt.subplots(figsize=(10, 5))\n",
+    "                for g_idx in [0, 1, 2]:\n",
+    "                    ax.plot(cons[mod][:, cell, g_idx], label=f\"G{g_idx} {cname} map\")\n",
+    "\n",
+    "                ax.set_xlabel(\"Stripes #\")\n",
+    "                ax.set_xlabel(cname)\n",
+    "                ax.set_title(f\"{cname} map - Cell {cell} - Module {mod} ({pdu})\")\n",
+    "                ax.legend()\n",
+    "                plt.show()"
    ]
   },
   {
@@ -480,7 +497,7 @@
    "outputs": [],
    "source": [
     "step_timer.start()\n",
-    "for mod, db_mod in zip(karabo_da, db_modules):\n",
+    "for mod, db_mod in da_to_pdu.items():\n",
     "    constants = {\n",
     "        \"Offset\": offset_map[mod],\n",
     "        \"Noise\": noise_map[mod],\n",
@@ -531,6 +548,26 @@
     ")\n",
     "step_timer.done_step(\"Injecting constants.\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "98ca9486",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# TODO: store old constants for comparison.\n",
+    "for mod, pdu in da_to_pdu.items():\n",
+    "    mod_file = mod.replace(\"/\", \"-\")\n",
+    "    with open(f\"{metadata_folder or out_folder}/module_metadata_{mod_file}.yml\", \"w\") as fd:\n",
+    "        yaml.safe_dump(\n",
+    "            {\n",
+    "                \"module\": mod,\n",
+    "                \"pdu\": pdu,\n",
+    "            },\n",
+    "            fd,\n",
+    "        )"
+   ]
   }
  ],
  "metadata": {