From 58845f6fd7bb728fb0687e31ac078658551bd911 Mon Sep 17 00:00:00 2001 From: ahmedk <karim.ahmed@xfel.eu> Date: Wed, 14 Dec 2022 11:12:44 +0100 Subject: [PATCH] remove Gotthard2 changes --- .../Gotthard2/Correction_Gotthard2_NBC.ipynb | 188 ++++++++---------- ...retrieve_constants_precorrection_NBC.ipynb | 159 ++++++++------- 2 files changed, 165 insertions(+), 182 deletions(-) diff --git a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb index 7bf1e4136..fc8913230 100644 --- a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb +++ b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb @@ -36,9 +36,10 @@ "karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n", "\n", "# Parameters for calibration database.\n", + "use_dir_creation_date = True # use the creation data of the input dir for database queries.\n", "cal_db_interface = \"tcp://max-exfl016:8016#8025\" # the database interface to use.\n", "cal_db_timeout = 180000 # timeout on caldb requests.\n", - "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n", + "overwrite_creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n", "\n", "# Parameters affecting corrected data.\n", "constants_file = \"\" # Use constants in given constant file path. /gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\n", @@ -69,8 +70,9 @@ "metadata": {}, "outputs": [], "source": [ + "import datetime\n", "import warnings\n", - "from logging import warning\n", + "from functools import partial\n", "\n", "import h5py\n", "import pasha as psh\n", @@ -80,16 +82,16 @@ "from extra_data import RunDirectory, H5File\n", "from pathlib import Path\n", "\n", - "from calibration_client import CalibrationClient\n", "from cal_tools import h5_copy_except\n", - "from cal_tools.calcat_interface import CalCatError, GOTTHARD2_CalibrationData\n", "from cal_tools.gotthard2 import gotthard2algs, gotthard2lib\n", - "from cal_tools.restful_config import restful_config\n", "from cal_tools.step_timing import StepTimer\n", "from cal_tools.tools import (\n", - " calcat_creation_time,\n", + " get_constant_from_db_and_time,\n", + " get_dir_creation_date,\n", + " get_pdu_from_db,\n", " CalibrationMetadata,\n", ")\n", + "from iCalibrationDB import Conditions, Constants\n", "from XFELDetAna.plotting.heatmap import heatmapPlot\n", "\n", "warnings.filterwarnings('ignore')\n", @@ -121,9 +123,14 @@ "\n", "print(f\"Process modules: {karabo_da} for run {run}\")\n", "\n", - "# Run's creation time:\n", - "creation_time = calcat_creation_time(in_folder, run, creation_time)\n", - "print(f\"Creation time: {creation_time}\")" + "creation_time = None\n", + "if overwrite_creation_time:\n", + " creation_time = datetime.datetime.strptime(\n", + " overwrite_creation_time, \"%Y-%m-%d %H:%M:%S.%f\"\n", + " )\n", + "elif use_dir_creation_date:\n", + " creation_time = get_dir_creation_date(in_folder, run)\n", + " print(f\"Using {creation_time} as creation time\")" ] }, { @@ -193,27 +200,6 @@ "print(\"Single Photon:\", single_photon)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "b3613034", - "metadata": {}, - "outputs": [], - "source": [ - "# Connect to CalCat.\n", - "calcat_config = restful_config['calcat']\n", - "client = CalibrationClient(\n", - " base_api_url=calcat_config['base-api-url'],\n", - " use_oauth2=calcat_config['use-oauth2'],\n", - " client_id=calcat_config['user-id'],\n", - " client_secret=calcat_config['user-secret'],\n", - " user_email=calcat_config['user-email'],\n", - " token_url=calcat_config['token-url'],\n", - " refresh_url=calcat_config['refresh-url'],\n", - " auth_url=calcat_config['auth-url'],\n", - " scope='')" - ] - }, { "cell_type": "markdown", "id": "8c852392-bb19-4c40-b2ce-3b787538a92d", @@ -225,33 +211,12 @@ { "cell_type": "code", "execution_count": null, - "id": "1817d0d8", + "id": "5717d722", "metadata": {}, "outputs": [], "source": [ - "da_to_pdu = {}\n", "# Used for old FXE (p003225) runs before adding Gotthard2 to CALCAT\n", "const_data = dict()\n", - "\n", - "g2_cal = GOTTHARD2_CalibrationData(\n", - " detector_name=karabo_id,\n", - " sensor_bias_voltage=bias_voltage,\n", - " exposure_time=exposure_time,\n", - " exposure_period=exposure_period,\n", - " acquisition_rate=acquisition_rate,\n", - " single_photon=single_photon,\n", - " event_at=creation_time,\n", - " client=client,\n", - ")\n", - "# Keep as long as it is essential to correct\n", - "# RAW data (FXE p003225) before the data mapping was added to CALCAT.\n", - "try: # in case local constants are used with old RAW data. This can be removed in the future.\n", - " for mod_info in g2_cal.physical_detector_units.values():\n", - " da_to_pdu[mod_info[\"karabo_da\"]] = mod_info[\"physical_name\"]\n", - "except CalCatError as e:\n", - " print(e)\n", - " db_modules = [None] * len(karabo_da)\n", - "\n", "if constants_file:\n", " for mod in karabo_da:\n", " const_data[mod] = dict()\n", @@ -260,59 +225,80 @@ " const_data[mod][\"LUT\"] = cfile[\"LUT\"][()]\n", " const_data[mod][\"Offset\"] = cfile[\"offset_map\"][()].astype(np.float32)\n", " const_data[mod][\"RelativeGain\"] = cfile[\"gain_map\"][()].astype(np.float32)\n", - " const_data[mod][\"Mask\"] = cfile[\"bpix_ff\"][()].astype(np.uint32)\n", - "else:\n", - " if const_yaml:\n", - " const_data = dict()\n", - " for mod in karabo_da:\n", - " const_data[mod] = dict()\n", + " const_data[mod][\"Mask\"] = cfile[\"bpix_ff\"][()].astype(np.uint32)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1cdbe818", + "metadata": {}, + "outputs": [], + "source": [ + "# Conditions iCalibrationDB object.\n", + "condition = Conditions.Dark.Gotthard2(\n", + " bias_voltage=bias_voltage,\n", + " exposure_time=exposure_time,\n", + " exposure_period=exposure_period,\n", + " single_photon=single_photon,\n", + " acquisition_rate=acquisition_rate,\n", + ")\n", + "\n", + "# TODO: Maybe this condition and previous cell can be removed later after the initial phase.\n", + "if not constants_file:\n", + " # Prepare a dictionary of empty constants to loop on\n", + " # it's keys and initiate non-retrieved constants.\n", + " empty_lut = (np.arange(2 ** 12).astype(np.float64) * 2 ** 10 / 2 ** 12).astype(\n", + " np.uint16\n", + " )\n", + " empty_lut = np.stack(1280 * [np.stack([empty_lut] * 2)], axis=0)\n", + " empty_constants = {\n", + " \"LUT\": empty_lut,\n", + " \"Offset\": np.zeros((1280, 2, 3), dtype=np.float32),\n", + " \"BadPixelsDark\": np.zeros((1280, 2, 3), dtype=np.uint32),\n", + " \"RelativeGain\": np.ones((1280, 2, 3), dtype=np.float32),\n", + " \"BadPixelsFF\": np.zeros((1280, 2, 3), dtype=np.uint32),\n", + " }\n", + "\n", + " for mod in karabo_da:\n", + " const_data[mod] = dict()\n", + " # Only used for printing timestamps within the loop.\n", + " when = dict()\n", + " # Check YAML file for constant metadata of file path and creation-time\n", + " if const_yaml:\n", " for cname, mdata in const_yaml[mod][\"constants\"].items():\n", " const_data[mod][cname] = dict()\n", - " if mdata[\"creation-time\"]:\n", - " with h5py.File(mdata[\"path\"], \"r\") as cf:\n", + " when[cname] = mdata[\"creation-time\"]\n", + " if when[cname]:\n", + " with h5py.File(mdata[\"file-path\"], \"r\") as cf:\n", " const_data[mod][cname] = np.copy(\n", - " cf[f\"{mdata['dataset']}/data\"])\n", - " else:\n", - " g2_cal = GOTTHARD2_CalibrationData(\n", - " detector_name=karabo_id,\n", - " sensor_bias_voltage=bias_voltage,\n", - " exposure_time=exposure_time,\n", - " exposure_period=exposure_period,\n", - " acquisition_rate=acquisition_rate,\n", - " single_photon=single_photon,\n", - " event_at=creation_time,\n", - " snapshot_at=creation_time,\n", - " )\n", - "\n", - " mdata_dict = {\"constants\": dict()}\n", - "\n", - " constant_names = [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"]\n", - " if gain_correction:\n", - " constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n", - "\n", - " # Retrieve metadata for all pnccd constants.\n", - "\n", - " g2_metadata = g2_cal.ndarray_map(constant_names)\n", - "\n", - " # Validate the constants availability and raise/warn correspondingly.\n", - " for mod, calibrations in const_data.items():\n", - " missing_dark_constants = set(\n", - " c for c in [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"] if c not in calibrations.keys())\n", - " missing_gain_constants = set(\n", - " c for c in [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"] if gain_correction and c not in calibrations.keys()) # noqa\n", - " if missing_dark_constants:\n", - " raise KeyError(\n", - " f\"Dark constants {missing_dark_constants} are not available for correction. Mod: {mod}\")\n", - "\n", - " if gain_correction and missing_gain_constants:\n", - " warning(f\"Gain constants {missing_gain_constants} are not retrieved. Mod {mod}\")\n", - " gain_correction = False\n", - "\n", - " # Create the mask array.\n", + " cf[f\"{mdata['dataset-name']}/data\"]\n", + " )\n", + " else:\n", + " const_data[mod][cname] = empty_constants[cname]\n", + " else: # Retrieve constants from CALCAT. Missing YAML file or running notebook interactively.\n", + " for cname, cempty in empty_constants.items():\n", + " const_data[mod][cname] = dict()\n", + " const_data[mod][cname], when[cname] = get_constant_from_db_and_time(\n", + " karabo_id=karabo_id,\n", + " karabo_da=mod,\n", + " cal_db_interface=cal_db_interface,\n", + " creation_time=creation_time,\n", + " timeout=cal_db_timeout,\n", + " print_once=False,\n", + " condition=condition,\n", + " constant=getattr(Constants.Gotthard2, cname)(),\n", + " empty_constant=cempty,\n", + " )\n", " bpix = const_data[mod][\"BadPixelsDark\"]\n", - " if const_data[mod].get(\"BadPixelsFF\") is not None:\n", - " bpix |= const_data[mod][\"BadPixelsFF\"]\n", - " const_data[mod][\"Mask\"] = bpix" + " bpix |= const_data[mod][\"BadPixelsFF\"]\n", + " const_data[mod][\"Mask\"] = bpix\n", + "\n", + " # Print timestamps for the retrieved constants.\n", + " print(f\"Constants for module {mod}:\")\n", + " for cname, ctime in when.items():\n", + " print(f\" {cname} injected at {ctime}\")\n", + " del when" ] }, { diff --git a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb index 6effefce9..18aa31302 100644 --- a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb +++ b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb @@ -32,12 +32,13 @@ "karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n", "\n", "# Parameters for calibration database.\n", - "cal_db_interface = \"tcp://max-exfl016:8017#8025\" # the database interface to use.\n", + "use_dir_creation_date = True # use the creation data of the input dir for database queries.\n", + "cal_db_interface = \"tcp://max-exfl017:8017#8025\" # the database interface to use.\n", "cal_db_timeout = 180000 # timeout on caldb requests.\n", - "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n", + "overwrite_creation_time = \"2022-06-28 13:00:00.00\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n", "\n", "# Parameters affecting corrected data.\n", - "constants_file = \"\" # /gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\" # Retrieve constants from local.\n", + "constants_file = \"\"#/gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\" # Retrieve constants from local.\n", "offset_correction = True # apply offset correction. This can be disabled to only apply LUT or apply LUT and gain correction for non-linear differential results.\n", "gain_correction = True # apply gain correction.\n", "\n", @@ -45,8 +46,8 @@ "bias_voltage = -1 # Detector bias voltage, set to -1 to use value in raw file.\n", "exposure_time = -1. # Detector exposure time, set to -1 to use value in raw file.\n", "exposure_period = -1. # Detector exposure period, set to -1 to use value in raw file.\n", - "acquisition_rate = -1. # Detector acquisition rate (1.1/4.5), set to -1 to use value in raw file.\n", - "single_photon = -1 # Detector single photon mode (High/Low CDS), set to -1 to use value in raw file.\n", + "acquisition_rate = 1.1 # Detector acquisition rate (1.1/4.5), set to -1 to use value in raw file.\n", + "single_photon = 0 # Detector single photon mode (High/Low CDS), set to -1 to use value in raw file.\n", "\n", "if constants_file:\n", " print(\"Skipping constant retrieval. Specified constants_file is used.\")\n", @@ -61,20 +62,20 @@ "metadata": {}, "outputs": [], "source": [ - "from logging import warning\n", + "import datetime\n", + "from functools import partial\n", "\n", + "import multiprocessing\n", "from extra_data import RunDirectory\n", "from pathlib import Path\n", "\n", - "from calibration_client import CalibrationClient\n", - "from cal_tools.calcat_interface import GOTTHARD2_CalibrationData\n", "from cal_tools.gotthard2 import gotthard2lib\n", - "from cal_tools.restful_config import restful_config\n", "from cal_tools.tools import (\n", - " calcat_creation_time,\n", + " get_dir_creation_date,\n", " get_from_db,\n", " CalibrationMetadata,\n", - ")" + ")\n", + "from iCalibrationDB import Conditions, Constants" ] }, { @@ -88,8 +89,6 @@ "out_folder = Path(out_folder)\n", "out_folder.mkdir(parents=True, exist_ok=True)\n", "metadata = CalibrationMetadata(metadata_folder or out_folder)\n", - "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n", - "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n", "\n", "if not karabo_id_control:\n", " karabo_id_control = karabo_id\n", @@ -99,9 +98,14 @@ "\n", "print(f\"Retrieve constants for modules: {karabo_da} for run {run}\")\n", "\n", - "# Run's creation time:\n", - "creation_time = calcat_creation_time(in_folder, run, creation_time)\n", - "print(f\"Creation time: {creation_time}\")" + "creation_time = None\n", + "if overwrite_creation_time:\n", + " creation_time = datetime.datetime.strptime(\n", + " overwrite_creation_time, \"%Y-%m-%d %H:%M:%S.%f\"\n", + " )\n", + "elif use_dir_creation_date:\n", + " creation_time = get_dir_creation_date(in_folder, run)\n", + " print(f\"Using {creation_time} as creation time\")" ] }, { @@ -138,73 +142,60 @@ "metadata": {}, "outputs": [], "source": [ - "# Connect to CalCat.\n", - "calcat_config = restful_config['calcat']\n", - "client = CalibrationClient(\n", - " base_api_url=calcat_config['base-api-url'],\n", - " use_oauth2=calcat_config['use-oauth2'],\n", - " client_id=calcat_config['user-id'],\n", - " client_secret=calcat_config['user-secret'],\n", - " user_email=calcat_config['user-email'],\n", - " token_url=calcat_config['token-url'],\n", - " refresh_url=calcat_config['refresh-url'],\n", - " auth_url=calcat_config['auth-url'],\n", - " scope='')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "g2_cal = GOTTHARD2_CalibrationData(\n", - " detector_name=karabo_id,\n", - " sensor_bias_voltage=bias_voltage,\n", + "condition = Conditions.Dark.Gotthard2(\n", + " bias_voltage=bias_voltage,\n", " exposure_time=exposure_time,\n", " exposure_period=exposure_period,\n", - " acquisition_rate=acquisition_rate,\n", " single_photon=single_photon,\n", - " event_at=creation_time,\n", - " snapshot_at=creation_time,\n", - " client=client,\n", + " acquisition_rate=acquisition_rate,\n", ")\n", "\n", - "mdata_dict = {\"constants\": dict()}\n", - "\n", - "constant_names = [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"]\n", - "if gain_correction:\n", - " constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n", - "\n", - "# Retrieve metadata for all pnccd constants.\n", + "def get_constants_for_module(mod: str):\n", + " \"\"\"Get calibration constants for given module for Gotthard2.\"\"\"\n", + " retrieval_function = partial(\n", + " get_from_db,\n", + " karabo_id=karabo_id,\n", + " karabo_da=mod,\n", + " cal_db_interface=cal_db_interface,\n", + " creation_time=creation_time,\n", + " timeout=cal_db_timeout,\n", + " verbosity=1,\n", + " meta_only=True,\n", + " load_data=False,\n", + " empty_constant=None\n", + " )\n", "\n", - "g2_metadata = g2_cal.metadata(constant_names)\n", - "# Validate the constants availability and raise/warn correspondingly.\n", - "for mod, ccv_dict in g2_metadata.items():\n", - " missing_dark_constants = set(\n", - " c for c in [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"] if c not in ccv_dict.keys())\n", - " missing_gain_constants = set(\n", - " c for c in [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"] if gain_correction and c not in ccv_dict.keys()) # noqa\n", - " if missing_dark_constants:\n", - " raise KeyError(\n", - " f\"Dark constants {missing_dark_constants} are not available for correction. Mod: {mod}\")\n", - " \n", - " if gain_correction and missing_gain_constants:\n", - " warning(f\"Gain constants {missing_gain_constants} are not retrieved. Mod {mod}\")\n", + " mdata_dict = dict()\n", + " mdata_dict[\"constants\"] = dict()\n", + " constants = [\n", + " \"LUT\", \"Offset\", \"BadPixelsDark\",\n", + " \"RelativeGain\", \"BadPixelsFF\",\n", + " ]\n", + " for cname in constants:\n", + " mdata_dict[\"constants\"][cname] = dict()\n", + " if not gain_correction and cname in [\"BadPixelsFF\", \"RelativeGain\"]:\n", + " continue\n", + " _, mdata = retrieval_function(\n", + " condition=condition,\n", + " constant=getattr(Constants.Gotthard2, cname)(),\n", + " )\n", + " mdata_const = mdata.calibration_constant_version\n", + " const_mdata = mdata_dict[\"constants\"][cname]\n", + " # check if constant was successfully retrieved.\n", + " if mdata.comm_db_success:\n", + " const_mdata[\"file-path\"] = (\n", + " f\"{mdata_const.hdf5path}\" f\"{mdata_const.filename}\"\n", + " )\n", + " const_mdata[\"dataset-name\"] = mdata_const.h5path\n", + " const_mdata[\"creation-time\"] = f\"{mdata_const.begin_at}\"\n", + " mdata_dict[\"physical-detector-unit\"] = mdata_const.device_name\n", + " else:\n", + " const_mdata[\"file-path\"] = None\n", + " const_mdata[\"creation-time\"] = None\n", + " return mdata_dict, mod\n", "\n", - "# Add constants metadata in retrieved_constants dict.\n", - "for mod, ccv_dict in g2_metadata.items():\n", - " mod_dict = retrieved_constants.setdefault(mod, dict())\n", - " const_dict = mod_dict.setdefault(\"constants\", dict())\n", - " for cname, ccv_metadata in ccv_dict.items():\n", - " const_dict[cname] = {\n", - " \"path\": str(g2_cal.caldb_root / ccv_metadata[\"path\"]),\n", - " \"dataset\": ccv_metadata[\"dataset\"],\n", - " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n", - " }\n", - " mod_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n", - " \n", - "print(f\"Stored retrieved constants in {metadata.filename}\")" + "with multiprocessing.Pool() as pool:\n", + " results = pool.map(get_constants_for_module, karabo_da)" ] }, { @@ -213,20 +204,26 @@ "metadata": {}, "outputs": [], "source": [ + "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n", + "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n", "timestamps = dict()\n", "\n", - "for mod in karabo_da:\n", + "for md_dict, mod in results:\n", + " if mod in retrieved_constants:\n", + " print(f\"Constant for {mod} already in calibration_metadata.yml, won't query again.\") # noqa\n", + " continue\n", + " retrieved_constants[mod] = md_dict\n", " module_timestamps = timestamps[mod] = dict()\n", - " module_constants = retrieved_constants[mod]\n", + "\n", " print(f\"Module: {mod}:\")\n", - " for cname, mdata in module_constants[\"constants\"].items():\n", + " for cname, mdata in md_dict[\"constants\"].items():\n", " if hasattr(mdata[\"creation-time\"], 'strftime'):\n", " mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n", " print(f'{cname:.<12s}', mdata[\"creation-time\"])\n", "\n", " for cname in [\"Offset\", \"BadPixelsDark\", \"RelativeGain\", \"BadPixelsFF\"]:\n", - " if cname in module_constants[\"constants\"]:\n", - " module_timestamps[cname] = module_constants[\"constants\"][cname][\"creation-time\"]\n", + " if cname in md_dict[\"constants\"]:\n", + " module_timestamps[cname] = md_dict[\"constants\"][cname][\"creation-time\"]\n", " else:\n", " module_timestamps[cname] = \"NA\"\n", "\n", -- GitLab