From 5cf97c958fd83679981ca3f365042bbff6caca23 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Fri, 5 Aug 2022 15:56:31 +0200
Subject: [PATCH] fixes after testing with reference run and remove unneeded
parameters in 1st cell
---
notebooks/LPD/LPD_Correct_Fast.ipynb | 18 +++----
...LPD_retrieve_constants_precorrection.ipynb | 53 +++++++------------
2 files changed, 27 insertions(+), 44 deletions(-)
diff --git a/notebooks/LPD/LPD_Correct_Fast.ipynb b/notebooks/LPD/LPD_Correct_Fast.ipynb
index e415151f4..39941d963 100644
--- a/notebooks/LPD/LPD_Correct_Fast.ipynb
+++ b/notebooks/LPD/LPD_Correct_Fast.ipynb
@@ -35,7 +35,7 @@
"output_source = '' # Output fast data source, empty to use same as input.\n",
"\n",
"# CalCat parameters\n",
- "use_dir_creation_date = True # Use the creation date of the directory for database time derivation.\n",
+ "creation_time = \"\" # The timestamp to use with Calibration DBe. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
"cal_db_interface = '' # Not needed, compatibility with current webservice.\n",
"cal_db_timeout = 0 # Not needed, compatbility with current webservice.\n",
"cal_db_root = '/gpfs/exfel/d/cal/caldb_store'\n",
@@ -105,7 +105,12 @@
"from extra_data.components import LPD1M\n",
"\n",
"from cal_tools.lpdalgs import correct_lpd_frames\n",
- "from cal_tools.tools import CalibrationMetadata, get_dir_creation_date, write_compressed_frames\n",
+ "from cal_tools.tools import (\n",
+ " CalibrationMetadata,\n",
+ " calcat_creation_time,\n",
+ " get_dir_creation_date,\n",
+ " write_compressed_frames,\n",
+ " )\n",
"from cal_tools.files import DataFile\n",
"from cal_tools.restful_config import restful_config"
]
@@ -135,12 +140,7 @@
"\n",
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"\n",
- "if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run) \n",
- "else:\n",
- " from datetime import datetime\n",
- " creation_time = datetime.now()\n",
- " \n",
+ "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
"print(f'Using {creation_time.isoformat()} as creation time')\n",
"\n",
"# Pick all modules/aggregators or those selected.\n",
@@ -242,7 +242,7 @@
" start = perf_counter()\n",
" for da, ccvs in const_yaml.items():\n",
"\n",
- " for calibration_name, ccv in ccvs.items():\n",
+ " for calibration_name, ccv in ccvs['constants'].items():\n",
"\n",
" dtype = np.uint32 if calibration_name.startswith('BadPixels') else np.float32\n",
"\n",
diff --git a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
index de8bae278..0864b34ce 100644
--- a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
+++ b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
@@ -32,7 +32,7 @@
"output_source = '' # Output fast data source, empty to use same as input.\n",
"\n",
"# CalCat parameters\n",
- "use_dir_creation_date = True # Use the creation date of the directory for database time derivation.\n",
+ "creation_time = \"\" # The timestamp to use with Calibration DBe. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
"cal_db_root = '/gpfs/exfel/d/cal/caldb_store'\n",
"\n",
"# Operating conditions\n",
@@ -40,23 +40,7 @@
"bias_voltage = 250.0 # Detector bias voltage.\n",
"capacitor = '5pF' # Capacitor setting: 5pF or 50pF\n",
"photon_energy = 9.2 # Photon energy in keV.\n",
- "category = 0 # Whom to blame.\n",
- "\n",
- "# Correction parameters\n",
- "offset_corr = True # Offset correction.\n",
- "rel_gain = True # Gain correction based on RelativeGain constant.\n",
- "ff_map = True # Gain correction based on FFMap constant.\n",
- "gain_amp_map = True # Gain correction based on GainAmpMap constant.\n",
- "\n",
- "# Output options\n",
- "overwrite = True # set to True if existing data should be overwritten\n",
- "chunks_data = 1 # HDF chunk size for pixel data in number of frames.\n",
- "chunks_ids = 32 # HDF chunk size for cellId and pulseId datasets.\n",
- "\n",
- "# Parallelization options\n",
- "max_nodes = 8 # Maximum number of SLURM jobs to split correction work into\n",
- "num_workers = 8 # Worker processes per node, 8 is safe on 768G nodes but won't work on 512G.\n",
- "num_threads_per_worker = 32 # Number of threads per worker."
+ "category = 0 # Whom to blame."
]
},
{
@@ -80,6 +64,7 @@
"\n",
"from cal_tools.tools import (\n",
" CalibrationMetadata,\n",
+ " calcat_creation_time,\n",
" get_dir_creation_date,\n",
" save_constant_metadata,\n",
" write_compressed_frames,\n",
@@ -108,12 +93,7 @@
"# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
"retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
"\n",
- "if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run) \n",
- "else:\n",
- " from datetime import datetime\n",
- " creation_time = datetime.now()\n",
- " \n",
+ "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
"print(f'Using {creation_time.isoformat()} as creation time')\n",
"\n",
"# Pick all modules/aggregators or those selected.\n",
@@ -160,7 +140,7 @@
"outputs": [],
"source": [
"dark_calibrations = {\n",
- " 11: 'Offset',\n",
+ " 1: 'Offset',\n",
" 14: 'BadPixelsDark',\n",
"}\n",
"\n",
@@ -174,9 +154,9 @@
"\n",
"illuminated_calibrations = {\n",
" 20: 'BadPixelsFF',\n",
- " 432: 'GainAmpMap',\n",
- " 413: 'FFMap',\n",
- " 424: 'RelativeGain',\n",
+ " 42: 'GainAmpMap',\n",
+ " 43: 'FFMap',\n",
+ " 44: 'RelativeGain',\n",
"}\n",
"\n",
"illuminated_condition = dark_condition.copy()\n",
@@ -209,17 +189,20 @@
" if not resp[\"success\"]:\n",
" print(f\"ERROR: Constants {list(calibrations.values())} \"\n",
" f\"were not retrieved, {resp['app_info']}\")\n",
- " const_mdata[\"file-path\"] = None\n",
- " const_mdata[\"dataset-name\"] = None\n",
- " const_mdata[\"creation-time\"] = None \n",
+ " for cname in calibrations.values():\n",
+ " const_mdata[cname] = dict()\n",
+ " const_mdata[cname][\"file-path\"] = None\n",
+ " const_mdata[cname][\"dataset-name\"] = None\n",
+ " const_mdata[cname][\"creation-time\"] = None \n",
" continue\n",
"\n",
" for ccv in resp[\"data\"]:\n",
" cc = ccv['calibration_constant']\n",
- " calibration_name = calibrations[cc['calibration_id']]\n",
- " const_mdata[\"file-path\"] = str(Path(ccv['path_to_file']) / ccv['file_name'])\n",
- " const_mdata[\"dataset-name\"] = ccv['data_set_name']\n",
- " const_mdata[\"creation-time\"] = ccv['begin_at']\n",
+ " cname = calibrations[cc['calibration_id']]\n",
+ " const_mdata[cname] = dict()\n",
+ " const_mdata[cname][\"file-path\"] = str(Path(ccv['path_to_file']) / ccv['file_name'])\n",
+ " const_mdata[cname][\"dataset-name\"] = ccv['data_set_name']\n",
+ " const_mdata[cname][\"creation-time\"] = ccv['begin_at']\n",
" pdu = ccv['physical_detector_unit']['physical_name']\n",
"\n",
" print('.', end='', flush=True)\n",
--
GitLab