[JUNGFRAU][CORRECT] Add fragment file and remove precorrection notebook

Add fragment file and remove precorrection notebook from JUNGFRA correction.

Description

• remove precorrection notebook.
• copy write_constants_fragment from !828 (merged)
• add sequence number for validation against values of reference corrected runs.

How Has This Been Tested?

Relevant Documents (optional)

Types of changes

• New feature (non-breaking change which adds functionality)

Checklist:

Reviewers

@kluyvert @schmidtp

notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb

@@ -79,7 +79,6 @@
     "from logging import warning\n",
     "from pathlib import Path\n",
     "\n",
-    "import h5py\n",
     "import matplotlib\n",
     "import matplotlib.pyplot as plt\n",
     "import numpy as np\n",
@@ -100,7 +99,7 @@
     "from cal_tools.tools import (\n",
     "    calcat_creation_time,\n",
     "    map_seq_files,\n",
-    "    CalibrationMetadata,\n",
+    "    write_constants_fragment,\n",
     ")\n",
     "\n",
     "warnings.filterwarnings('ignore')\n",
@@ -121,10 +120,6 @@
     "run_dc = RunDirectory(run_folder)\n",
     "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
     "\n",
-    "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
-    "# NOTE: this notebook will not overwrite calibration metadata file\n",
-    "const_yaml = metadata.get(\"retrieved-constants\", {})\n",
-    "\n",
     "out_folder.mkdir(parents=True, exist_ok=True)\n",
     "\n",
     "print(f\"Run is: {run}\")\n",
@@ -178,7 +173,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -207,24 +201,11 @@
     "for mod_info in jf_cal.physical_detector_units.values():\n",
     "    da_to_pdu[mod_info[\"karabo_da\"]] = mod_info[\"physical_name\"]\n",
     "\n",
-    "if const_yaml:\n",
-    "    const_data = dict()\n",
-    "    for mod, constants in const_yaml.items():\n",
-    "        if mod not in karabo_da:\n",
-    "            continue  # skip other keys like time-summary\n",
-    "        const_data[mod] = dict()\n",
-    "        for cname, mdata in constants[\"constants\"].items():\n",
-    "            const_data[mod][cname] = dict()\n",
-    "            if mdata[\"creation-time\"]:\n",
-    "                with h5py.File(mdata[\"path\"], \"r\") as cf:\n",
-    "                    const_data[mod][cname] = np.copy(\n",
-    "                        cf[f\"{mdata['dataset']}/data\"])\n",
-    "else:\n",
-    "    constant_names = [\"Offset10Hz\", \"BadPixelsDark10Hz\"]\n",
-    "    if relative_gain:\n",
-    "        constant_names += [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"]\n",
+    "constant_names = [\"Offset10Hz\", \"BadPixelsDark10Hz\"]\n",
+    "if relative_gain:\n",
+    "    constant_names += [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"]\n",
     "\n",
-    "    const_data = jf_cal.ndarray_map(calibrations=constant_names)"
+    "jf_metadata = jf_cal.metadata(calibrations=constant_names)"
    ]
   },
   {
@@ -235,7 +216,7 @@
    "source": [
     "# Validate the constants availability and raise/warn correspondingly. \n",
     "for mod in karabo_da[:]:\n",
-    "    calibrations = const_data.get(mod, {})\n",
+    "    calibrations = jf_metadata.get(mod, {})\n",
     "\n",
     "    missing_dark_constants = {\"Offset10Hz\", \"BadPixelsDark10Hz\"} - set(calibrations)\n",
     "    missing_gain_constants = {\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"} - set(calibrations)\n",
@@ -253,6 +234,24 @@
     "    raise ValueError(\"Dark constants are missing for all modules.\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Record constant details in YAML metadata\n",
+    "write_constants_fragment(\n",
+    "    out_folder=(metadata_folder or out_folder),\n",
+    "    det_metadata=jf_metadata,\n",
+    "    caldb_root=jf_cal.caldb_root)\n",
+    "\n",
+    "\n",
+    "# load constants arrays after storing fragment YAML file\n",
+    "# and validating constants availability.\n",
+    "const_data = jf_cal.ndarray_map(metadata=jf_metadata)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -619,7 +618,10 @@
     "            save_reduced_rois(outp_file, data_corr, mask_corr, local_karabo_da)\n",
     "\n",
     "            # Create METDATA datasets\n",
-    "            outp_file.create_metadata(like=seq_dc)\n",
+    "            outp_file.create_metadata(\n",
+    "                like=seq_dc,\n",
+    "                sequence=seq_dc.run_metadata()[\"sequenceNumber\"],\n",
+    "            )\n",
     "\n",
     "        step_timer.done_step(f'Saving data time.')\n",
     "if empty_seq == sum([len(i) for i in mapped_files.values()]):\n",
@@ -1012,9 +1014,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "cal2_venv",
    "language": "python",
-   "name": "python3"
+   "name": "cal2_venv"
   },
   "language_info": {
    "codemirror_mode": {
@@ -1026,7 +1028,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.10"
+   "version": "3.8.11"
   }
  },
  "nbformat": 4,

[Karim Ahmed]

I just added this for the tests to succeed. I think I added this every time for the tests to pass but never pushed the change. I should have added this separately but I was lazy to add a line of code in a separate MR.