From e7631e0a2b9aa044e6d22c3c999c61b6737a50a0 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Wed, 14 Dec 2022 10:21:22 +0100
Subject: [PATCH] remove pnccd changes
---
notebooks/pnCCD/Correct_pnCCD_NBC.ipynb | 134 +++++++++---------
...CCD_retrieve_constants_precorrection.ipynb | 124 +++++++---------
2 files changed, 118 insertions(+), 140 deletions(-)
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index bf328394c..b479ac76a 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -94,7 +94,6 @@
"import datetime\n",
"import os\n",
"import warnings\n",
- "from logging import warning\n",
"from pathlib import Path\n",
"warnings.filterwarnings('ignore')\n",
"\n",
@@ -108,19 +107,21 @@
"\n",
"%matplotlib inline\n",
"\n",
- "from calibration_client import CalibrationClient\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
"from cal_tools import pnccdlib\n",
- "from cal_tools.calcat_interface import PNCCD_CalibrationData\n",
- "from cal_tools.restful_config import restful_config\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
- " load_constants_dict,\n",
+ " get_dir_creation_date,\n",
+ " get_constant_from_db_and_time,\n",
+ " get_random_db_interface,\n",
+ " load_specified_constants,\n",
" CalibrationMetadata,\n",
")\n",
"from cal_tools.step_timing import StepTimer\n",
- "from cal_tools import h5_copy_except"
+ "from cal_tools import h5_copy_except\n",
+ "from iCalibrationDB import Conditions, Constants\n",
+ "from iCalibrationDB.detectors import DetectorTypes"
]
},
{
@@ -300,26 +301,6 @@
"b_range = Event_Bin_Dict[\"b_range\"]"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Connect to CalCat.\n",
- "calcat_config = restful_config['calcat']\n",
- "client = CalibrationClient(\n",
- " base_api_url=calcat_config['base-api-url'],\n",
- " use_oauth2=calcat_config['use-oauth2'],\n",
- " client_id=calcat_config['user-id'],\n",
- " client_secret=calcat_config['user-secret'],\n",
- " user_email=calcat_config['user-email'],\n",
- " token_url=calcat_config['token-url'],\n",
- " refresh_url=calcat_config['refresh-url'],\n",
- " auth_url=calcat_config['auth-url'],\n",
- " scope='')"
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -335,38 +316,50 @@
"source": [
"display(Markdown(\"### Constants retrieval\"))\n",
"step_timer.start()\n",
- "constant_names = [\"OffsetCCD\", \"NoiseCCD\", \"BadPixelsDarkCCD\"]\n",
- "if relgain:\n",
- " constant_names += [\"RelativeGainCCD\"]\n",
+ "\n",
+ "conditions_dict = {\n",
+ " \"bias_voltage\": bias_voltage,\n",
+ " \"integration_time\": integration_time,\n",
+ " \"gain_setting\": gain,\n",
+ " \"temperature\": fix_temperature_top,\n",
+ " \"pixels_x\": pixels_x,\n",
+ " \"pixels_y\": pixels_y,\n",
+ "}\n",
+ "# Dark condition\n",
+ "dark_condition = Conditions.Dark.CCD(**conditions_dict)\n",
+ "# Add photon energy.\n",
+ "conditions_dict.update({\"photon_energy\": photon_energy})\n",
+ "illum_condition = Conditions.Illuminated.CCD(**conditions_dict)\n",
+ "\n",
+ "# A dictionary for initializing constants. {cname: empty constant array}\n",
+ "empty_constants = {\n",
+ " \"Offset\": np.zeros((pixels_x, pixels_y, 1), dtype=np.float32),\n",
+ " \"Noise\": np.zeros((pixels_x, pixels_y, 1), dtype=np.float32),\n",
+ " \"BadPixelsDark\": np.zeros((pixels_x, pixels_y, 1), dtype=np.uint32),\n",
+ " \"RelativeGain\": np.zeros((pixels_x, pixels_y), dtype=np.float32),\n",
+ "}\n",
"\n",
"if const_yaml: # Used while reproducing corrected data.\n",
" print(f\"Using stored constants in {metadata.filename}\")\n",
- " constants, _ = load_constants_dict(const_yaml[karabo_da][\"constants\"])\n",
- "else:\n",
- " pnccd_cal = PNCCD_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " sensor_temperature=fix_temperature_top,\n",
- " gain_setting=gain,\n",
- " event_at=creation_time,\n",
- " snapshot_at=creation_time,\n",
- " client=client,\n",
+ " constants, when = load_specified_constants(\n",
+ " const_yaml[karabo_da][\"constants\"], empty_constants\n",
" )\n",
- " constants = pnccd_cal.ndarray_map(calibrations=constant_names)[karabo_da]\n",
- "\n",
- "# Validate the constants availability and raise/warn correspondingly. \n",
- "missing_dark_constants = set(\n",
- " c for c in [\"OffsetCCD\", \"NoiseCCD\", \"BadPixelsDarkCCD\"] if c not in constants.keys())\n",
- "\n",
- "if missing_dark_constants:\n",
- " raise KeyError(\n",
- " f\"Dark constants {missing_dark_constants} are not available for correction.\")\n",
- "\n",
- "if corr_bools.get('relgain') and \"RelativeGainCCD\" not in constants.keys():\n",
- " warning(\"RelativeGainEPix100 is not found in the calibration database.\")\n",
- " corr_bools['relgain'] = False\n",
- "step_timer.done_step(\"Constants retrieval\")"
+ "else:\n",
+ " constants = dict()\n",
+ " when = dict()\n",
+ " for cname, cempty in empty_constants.items():\n",
+ " # No need for retrieving RelativeGain, if not used for correction.\n",
+ " if not corr_bools.get(\"relgain\") and cname == \"RelativeGain\":\n",
+ " continue\n",
+ " constants[cname], when[cname] = get_constant_from_db_and_time(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " constant=getattr(Constants.CCD(DetectorTypes.pnCCD), cname)(),\n",
+ " condition=illum_condition if cname == \"RelativeGain\" else dark_condition,\n",
+ " empty_constant=cempty,\n",
+ " cal_db_interface=get_random_db_interface(cal_db_interface),\n",
+ " creation_time=creation_time,\n",
+ " )"
]
},
{
@@ -375,32 +368,33 @@
"metadata": {},
"outputs": [],
"source": [
- "fig = xana.heatmapPlot(constants[\"OffsetCCD\"][:,:,0], x_label='Columns', y_label='Rows', lut_label='Offset (ADU)', \n",
+ "fig = xana.heatmapPlot(constants[\"Offset\"][:,:,0], x_label='Columns', y_label='Rows', lut_label='Offset (ADU)', \n",
" aspect=1, \n",
" x_range=(0, pixels_y), y_range=(0, pixels_x), vmax=16000, \n",
" panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', \n",
" title = 'Dark Offset Map')\n",
"\n",
- "fig = xana.heatmapPlot(constants[\"NoiseCCD\"][:,:,0], x_label='Columns', y_label='Rows', \n",
+ "fig = xana.heatmapPlot(constants[\"Noise\"][:,:,0], x_label='Columns', y_label='Rows', \n",
" lut_label='Corrected Noise (ADU)', \n",
" aspect=1, x_range=(0, pixels_y), y_range=(0, pixels_x), \n",
" panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', \n",
" title = 'Dark Noise Map')\n",
"\n",
- "fig = xana.heatmapPlot(np.log2(constants[\"BadPixelsDarkCCD\"][:,:,0]), x_label='Columns', y_label='Rows', \n",
+ "fig = xana.heatmapPlot(np.log2(constants[\"BadPixelsDark\"][:,:,0]), x_label='Columns', y_label='Rows', \n",
" lut_label='Bad Pixel Value (ADU)', \n",
" aspect=1, x_range=(0, pixels_y), y_range=(0, pixels_x), \n",
" panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', \n",
" title = 'Dark Bad Pixels Map')\n",
"\n",
"if corr_bools.get('relgain'):\n",
- " fig = xana.heatmapPlot(constants[\"RelativeGainCCD\"], figsize=(8, 8), x_label='Columns', y_label='Rows', \n",
+ " fig = xana.heatmapPlot(constants[\"RelativeGain\"], figsize=(8, 8), x_label='Columns', y_label='Rows', \n",
" lut_label='Relative Gain', \n",
" aspect=1, x_range=(0, pixels_y), y_range=(0, pixels_x), vmin=0.8, vmax=1.2, \n",
" panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', \n",
" panel_top_low_lim = 0.5, panel_top_high_lim = 1.5, panel_side_low_lim = 0.5, \n",
" panel_side_high_lim = 1.5, \n",
- " title = f'Relative Gain Map for pnCCD (Gain = 1/{int(gain)})')"
+ " title = f'Relative Gain Map for pnCCD (Gain = 1/{int(gain)})')\n",
+ "step_timer.done_step(\"Constants retrieval\")"
]
},
{
@@ -416,13 +410,13 @@
" commonModeBlockSize,\n",
" commonModeAxis,\n",
" parallel=False, dType=np.float32, stride=1,\n",
- " noiseMap=constants[\"NoiseCCD\"].astype(np.float32), minFrac=0.25)\n",
+ " noiseMap=constants[\"Noise\"].astype(np.float32), minFrac=0.25)\n",
"\n",
"if corr_bools.get('pattern_class'):\n",
" # Pattern Classifier Calculator:\n",
" # Left Hemisphere:\n",
" patternClassifierLH = xcal.PatternClassifier([pixels_x, pixels_y//2],\n",
- " constants[\"NoiseCCD\"][:, :pixels_y//2],\n",
+ " constants[\"Noise\"][:, :pixels_y//2],\n",
" split_evt_primary_threshold,\n",
" split_evt_secondary_threshold,\n",
" split_evt_mip_threshold,\n",
@@ -434,7 +428,7 @@
"\n",
" # Right Hemisphere:\n",
" patternClassifierRH = xcal.PatternClassifier([pixels_x, pixels_y//2],\n",
- " constants[\"NoiseCCD\"][:, pixels_y//2:],\n",
+ " constants[\"Noise\"][:, pixels_y//2:],\n",
" split_evt_primary_threshold,\n",
" split_evt_secondary_threshold,\n",
" split_evt_mip_threshold,\n",
@@ -447,11 +441,11 @@
" patternClassifierLH._imagesPerChunk = 1\n",
" patternClassifierRH._imagesPerChunk = 1\n",
"\n",
- " patternClassifierLH._noisemap = constants[\"NoiseCCD\"][:, :pixels_x//2]\n",
- " patternClassifierRH._noisemap = constants[\"NoiseCCD\"][:, pixels_x//2:]\n",
+ " patternClassifierLH._noisemap = constants[\"Noise\"][:, :pixels_x//2]\n",
+ " patternClassifierRH._noisemap = constants[\"Noise\"][:, pixels_x//2:]\n",
" # Setting bad pixels:\n",
- " patternClassifierLH.setBadPixelMask(constants[\"BadPixelsDarkCCD\"][:, :pixels_x//2] != 0)\n",
- " patternClassifierRH.setBadPixelMask(constants[\"BadPixelsDarkCCD\"][:, pixels_x//2:] != 0)"
+ " patternClassifierLH.setBadPixelMask(constants[\"BadPixelsDark\"][:, :pixels_x//2] != 0)\n",
+ " patternClassifierRH.setBadPixelMask(constants[\"BadPixelsDark\"][:, pixels_x//2:] != 0)"
]
},
{
@@ -591,10 +585,10 @@
"\n",
"data_path = \"INSTRUMENT/\"+instrument_src+\"/data/\"\n",
"\n",
- "offset = np.squeeze(constants[\"OffsetCCD\"])\n",
- "noise = np.squeeze(constants[\"NoiseCCD\"])\n",
- "bpix = np.squeeze(constants[\"BadPixelsDarkCCD\"])\n",
- "relativegain = constants.get(\"RelativeGainCCD\")"
+ "offset = np.squeeze(constants[\"Offset\"])\n",
+ "noise = np.squeeze(constants[\"Noise\"])\n",
+ "bpix = np.squeeze(constants[\"BadPixelsDark\"])\n",
+ "relativegain = constants.get(\"RelativeGain\")"
]
},
{
diff --git a/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb b/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
index 44a5b5934..eeac52fe3 100644
--- a/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
+++ b/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
@@ -21,15 +21,18 @@
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/pnccd_correct\" # output folder\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"run = 347 # which run to read data from\n",
+ "sequences = [0] # sequences to correct, set to -1 for all, range allowed\n",
"\n",
"karabo_da = 'PNCCD01' # data aggregators\n",
"karabo_id = \"SQS_NQS_PNCCD1MP\" # detector Karabo_ID\n",
"\n",
"# Conditions for retrieving calibration constants\n",
"fix_temperature_top = 0. # fix temperature for top sensor in K, set to 0. to use value from slow data.\n",
+ "fix_temperature_bot = 0. # fix temperature for bottom sensor in K, set to 0. to use value from slow data.\n",
"gain = -1 # the detector's gain setting. Set to -1 to use the value from the slow data.\n",
"bias_voltage = 0. # the detector's bias voltage. set to 0. to use value from slow data.\n",
"integration_time = 70 # detector's integration time\n",
+ "photon_energy = 1.6 # Al fluorescence in keV\n",
"\n",
"# Parameters for the calibration database.\n",
"cal_db_interface = \"tcp://max-exfl016:8015\" # calibration DB interface to use\n",
@@ -50,20 +53,23 @@
"metadata": {},
"outputs": [],
"source": [
- "from logging import warning\n",
+ "import datetime\n",
"from pathlib import Path\n",
"\n",
"from IPython.display import Markdown, display\n",
"from extra_data import RunDirectory\n",
"\n",
- "from calibration_client import CalibrationClient\n",
"from cal_tools import pnccdlib\n",
- "from cal_tools.calcat_interface import PNCCD_CalibrationData\n",
- "from cal_tools.restful_config import restful_config\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
+ " get_dir_creation_date,\n",
+ " get_from_db,\n",
+ " get_random_db_interface,\n",
+ " save_constant_metadata,\n",
" CalibrationMetadata,\n",
- ")"
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants\n",
+ "from iCalibrationDB.detectors import DetectorTypes"
]
},
{
@@ -72,15 +78,17 @@
"metadata": {},
"outputs": [],
"source": [
- "in_folder = Path(in_folder)\n",
- "out_folder = Path(out_folder)\n",
- "\n",
- "out_folder.mkdir(parents=True, exist_ok=True)\n",
- "\n",
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"# NOTE: this notebook will not overwrite calibration metadata file,\n",
"# if it already contains details about which constants to use.\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})"
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
+ "if karabo_da in retrieved_constants:\n",
+ " print(\n",
+ " f\"Constant for {karabo_da} already in {metadata.filename}, won't query again.\"\n",
+ " ) # noqa\n",
+ " import sys\n",
+ "\n",
+ " sys.exit(0)"
]
},
{
@@ -118,7 +126,7 @@
"metadata": {},
"outputs": [],
"source": [
- "run_dc = RunDirectory(in_folder / f\"r{run:04d}\", _use_voview=False)\n",
+ "run_dc = RunDirectory(Path(in_folder) / f\"r{run:04d}\", _use_voview=False)\n",
"ctrl_data = pnccdlib.PnccdCtrl(run_dc, karabo_id)\n",
"\n",
"# extract control data\n",
@@ -137,26 +145,6 @@
"print(f\"Top pnCCD sensor temperature: {fix_temperature_top:0.2f} K\")"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Connect to CalCat.\n",
- "calcat_config = restful_config['calcat']\n",
- "client = CalibrationClient(\n",
- " base_api_url=calcat_config['base-api-url'],\n",
- " use_oauth2=calcat_config['use-oauth2'],\n",
- " client_id=calcat_config['user-id'],\n",
- " client_secret=calcat_config['user-secret'],\n",
- " user_email=calcat_config['user-email'],\n",
- " token_url=calcat_config['token-url'],\n",
- " refresh_url=calcat_config['refresh-url'],\n",
- " auth_url=calcat_config['auth-url'],\n",
- " scope='')"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -164,45 +152,41 @@
"outputs": [],
"source": [
"display(Markdown(\"### Constants retrieval\"))\n",
- "pnccd_cal = PNCCD_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " sensor_temperature=fix_temperature_top,\n",
- " gain_setting=gain,\n",
- " event_at=creation_time,\n",
- " snapshot_at=creation_time,\n",
- " client=client,\n",
- ")\n",
- "\n",
- "mdata_dict = {\"constants\": dict()}\n",
- "\n",
- "constant_names = [\"OffsetCCD\", \"NoiseCCD\", \"BadPixelsDarkCCD\"]\n",
- "if relgain:\n",
- " constant_names += [\"RelativeGainCCD\"]\n",
- "\n",
- "# Retrieve metadata for all pnccd constants.\n",
- "\n",
- "pnccd_metadata = pnccd_cal.metadata(constant_names)\n",
- "\n",
- "# Validate the constants availability and raise/warn correspondingly. \n",
- "missing_dark_constants = set(\n",
- " c for c in [\"OffsetCCD\", \"NoiseCCD\", \"BadPixelsDarkCCD\"] if c not in pnccd_metadata[karabo_da].keys())\n",
- "if missing_dark_constants:\n",
- " raise KeyError(\n",
- " f\"Dark constants {missing_dark_constants} are not available for correction.\")\n",
- "if relgain and \"RelativeGainCCD\" not in list(pnccd_metadata.values())[0].keys():\n",
- " warning(\"RelativeGainCCD is not found in CALCAT.\")\n",
- "\n",
- "for cname, ccv_metadata in list(pnccd_metadata.values())[0].items():\n",
- " mdata_dict[\"constants\"][cname] = {\n",
- " \"path\": str(pnccd_cal.caldb_root / ccv_metadata[\"path\"]),\n",
- " \"dataset\": ccv_metadata[\"dataset\"],\n",
- " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
- " }\n",
- " print(f\"Retrieved {cname} with creation-time: {ccv_metadata['begin_validity_at']}\")\n",
"\n",
- "mdata_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
+ "conditions_dict = {\n",
+ " \"bias_voltage\": bias_voltage,\n",
+ " \"integration_time\": integration_time,\n",
+ " \"gain_setting\": gain,\n",
+ " \"temperature\": fix_temperature_top,\n",
+ " \"pixels_x\": 1024,\n",
+ " \"pixels_y\": 1024,\n",
+ "}\n",
+ "# Dark condition\n",
+ "dark_condition = Conditions.Dark.CCD(**conditions_dict)\n",
+ "# Add photon energy.\n",
+ "conditions_dict.update({\"photon_energy\": photon_energy})\n",
+ "illum_condition = Conditions.Illuminated.CCD(**conditions_dict)\n",
+ "\n",
+ "mdata_dict = dict()\n",
+ "mdata_dict[\"constants\"] = dict()\n",
+ "for cname in [\"Offset\", \"Noise\", \"BadPixelsDark\", \"RelativeGain\"]:\n",
+ " # No need for retrieving RelativeGain, if not used for correction.\n",
+ " if not corr_bools.get(\"relgain\") and cname == \"RelativeGain\":\n",
+ " continue\n",
+ " _, mdata = get_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " constant=getattr(Constants.CCD(DetectorTypes.pnCCD), cname)(),\n",
+ " condition=illum_condition if cname == \"RelativeGain\" else dark_condition,\n",
+ " empty_constant=None,\n",
+ " cal_db_interface=get_random_db_interface(cal_db_interface),\n",
+ " creation_time=creation_time,\n",
+ " verbosity=1,\n",
+ " load_data=False,\n",
+ " )\n",
+ " save_constant_metadata(mdata_dict[\"constants\"], mdata, cname)\n",
+ "\n",
+ "mdata_dict[\"physical-detector-unit\"] = mdata.calibration_constant_version.device_name\n",
"retrieved_constants[karabo_da] = mdata_dict\n",
"metadata.save()\n",
"print(f\"Stored retrieved constants in {metadata.filename}\")"
--
GitLab