diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index c253a7d7cfa978b5b5920d428cfabed35ba430ff..abed088acd87441711bf553242fe7346d6d56265 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -6,19 +6,21 @@ from os import environ, listdir, path, stat
from os.path import isfile, splitext
from queue import Queue
import re
+import textwrap
from time import sleep
from urllib.parse import urljoin
import dateutil.parser
-
import ipykernel
from metadata_client.metadata_client import MetadataClient
from notebook.notebookapp import list_running_servers
import numpy as np
import requests
-from .mdc_config import MDC_config
from .ana_tools import save_dict_to_hdf5
+from iCalibrationDB import ConstantMetaData, Versions
+from .mdc_config import MDC_config
+import zmq
def parse_runs(runs, return_type=str):
pruns = []
@@ -264,14 +266,39 @@ def get_dir_creation_date(directory, run, tsdir=False, verbosity=0):
ntries -= 1
-def save_const_to_h5(metadata, out_folder):
+def save_const_to_h5(device, constant, data, condition,
+ file_loc, creation_time, out_folder):
"""
Save constant in h5 file with its metadata
(e.g. db_module, condition, creation_time)
- :param metadata: Metadata
+ :param device: Instance of detector
+ :type device: iCalibrationDB.detectors object
+ :param constant: Calibration constant known for given detector
+ :type constant: iCalibrationDB.know_constants object
+ :param condition: Calibration condition
+ :type condition: iCalibrationDB.know_detector_conditions object
+ :param data: Constant data to save
+ :type data: ndarray
+ :param file_loc: Location of raw data "proposal:{} runs:{} {} {}"
+ :type file_loc: str
+ :param creation_time: creation_time for the saved constant
+ :type creation_time: datetime object
:param out_folder: path to output folder
+ :type out_folder: str
"""
+ metadata = ConstantMetaData()
+ metadata.calibration_constant = constant
+ metadata.detector_condition = condition
+ if creation_time is None:
+ metadata.calibration_constant_version = Versions.Now(
+ device=device)
+ else:
+ metadata.calibration_constant_version = Versions.Timespan(
+ device=device,
+ start=creation_time)
+
+ metadata.calibration_constant_version.raw_data_location = file_loc
dpar = {}
for parm in metadata.detector_condition.parameters:
@@ -289,15 +316,16 @@ def save_const_to_h5(metadata, out_folder):
data_to_store['condition'] = dpar
data_to_store['db_module'] = db_module
data_to_store['constant'] = constant_name
- data_to_store['data'] = metadata.calibration_constant.data
+ data_to_store['data'] = data
data_to_store['creation_time'] = creation_time
data_to_store['file_loc'] = raw_data
- ofile = "{}/const_{}_{}.h5".format(out_folder, constant_name, db_module)
+ ofile = f"{out_folder}/const_{constant_name}_{db_module}.h5"
if isfile(ofile):
- print('File {} already exists and will be overwritten'.format(ofile))
+ print(f'File {ofile} already exists and will be overwritten')
save_dict_to_hdf5(data_to_store, ofile)
+ return metadata
def get_random_db_interface(cal_db_interface):
"""
@@ -337,8 +365,6 @@ def get_from_db(device, constant, condition, empty_constant,
:param doraise: (bool) if True raise errors during communication with DB.
:return: Calibration constant, metadata
"""
- from iCalibrationDB import ConstantMetaData, Versions
- import zmq
if version_info:
meta_only = False
@@ -384,7 +410,6 @@ def get_from_db(device, constant, condition, empty_constant,
if ntries == 0 and doraise:
raise RuntimeError(f'{e}')
-
if ntries > 0:
if verbosity > 0:
if constant.name not in already_printed or verbosity > 1:
@@ -401,26 +426,33 @@ def get_from_db(device, constant, condition, empty_constant,
def send_to_db(device, constant, condition, file_loc,
cal_db_interface, creation_time=None,
- verbosity=1, timeout=30000, ntries=7, doraise=False):
+ timeout=30000, ntries=7, doraise=False):
"""
Return calibration constants and metadata requested from CalDB
:param device: Instance of detector
+ :type device: iCalibrationDB.detectors object
:param constant: Calibration constant known for given detector
+ :type constant: iCalibrationDB.know_constants object
:param condition: Calibration condition
+ :type condition: iCalibrationDB.know_detector_conditions object
:param file_loc: Location of raw data.
+ :type file_loc: str
:param cal_db_interface: Interface string, e.g. "tcp://max-exfl016:8015"
+ :type cal_db_interface: str
:param creation_time: Latest time for constant to be created
- :param verbosity: Level of verbosity (0 - silent)
+ :type creation_time: datetime object
:param timeout: Timeout for zmq request
+ :type timeout: int
:param ntries: number of tries to contact the database,
ntries is set to 7 so that if the timeout started at 30s last timeout
will be ~ 1h.
- :param doraise: (bool) if True raise errors during communication with DB
+ :type ntries: int
+ :param doraise: if True raise errors during communication with DB
+ :type doraise: bool
"""
- from iCalibrationDB import ConstantMetaData, Versions
- import zmq
+ success = False
if device:
metadata = ConstantMetaData()
metadata.calibration_constant = constant
@@ -440,6 +472,7 @@ def send_to_db(device, constant, condition, file_loc,
this_interface = get_random_db_interface(cal_db_interface)
try:
r = metadata.send(this_interface, timeout=timeout)
+ success = True
break
except zmq.error.Again:
ntries -= 1
@@ -448,8 +481,14 @@ def send_to_db(device, constant, condition, file_loc,
if ntries == 0 and doraise:
raise
except Exception as e:
- if verbosity > 0:
- print(e)
+ if "has already been take" in str(e):
+ print(f"WARNING: {constant.name} has already been "
+ "injected with the same parameter "
+ "conditions\n")
+ else:
+ # reduce error text if it is too long.
+ print("\n".join(textwrap.wrap(str(e), 100)))
+
if 'missing_token' in str(e):
ntries -= 1
else:
@@ -457,14 +496,10 @@ def send_to_db(device, constant, condition, file_loc,
if ntries == 0 and doraise:
raise RuntimeError(f'{e}')
- if ntries > 0:
- if verbosity > 0:
- if constant.name not in already_printed or verbosity > 1:
- already_printed[constant.name] = True
- begin_at = metadata.calibration_constant_version.begin_at
- print("{} was sent on: {}".format(constant.name,
- begin_at))
-
+ if success:
+ print(f"{constant.name} is injected with creation-time: "
+ f"{metadata.calibration_constant_version.begin_at}\n")
+ return metadata
def get_constant_from_db(device, constant, condition, empty_constant,
cal_db_interface, creation_time=None,
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index eaae915a6a8f7562452b0b4dd4f116b9f44be484..6e69bb8e79af6313e6c41b2fd0e3140ceb935f77 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -28,7 +28,7 @@
"in_folder = \"/gpfs/exfel/d/raw/SPB/202030/p900138\" # path to input data, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPDbad_sep64\" # path to output to, required\n",
"sequences = [0] # sequence files to evaluate.\n",
- "modules = [-1] # list of modules to evaluate, RANGE ALLOWED\n",
+ "modules = [0] # list of modules to evaluate, RANGE ALLOWED\n",
"run_high = 167 # run number in which high gain data was recorded, required\n",
"run_med = 168 # run number in which medium gain data was recorded, required\n",
"run_low = 169 # run number in which low gain data was recorded, required\n",
@@ -101,15 +101,16 @@
"from IPython.display import display, Markdown, Latex\n",
"%matplotlib inline\n",
"\n",
- "from cal_tools.tools import (map_gain_stages, parse_runs, \n",
- " run_prop_seq_from_path, get_notebook_name, \n",
- " get_dir_creation_date, save_const_to_h5,\n",
- " get_random_db_interface, get_from_db)\n",
+ "from cal_tools.tools import (get_from_db, get_dir_creation_date,\n",
+ " get_notebook_name, get_random_db_interface,\n",
+ " map_gain_stages, parse_runs,\n",
+ " run_prop_seq_from_path, save_const_to_h5,\n",
+ " send_to_db)\n",
"from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import (show_overview, plot_badpix_3d,\n",
- " create_constant_overview,\n",
- " show_processed_modules)\n",
+ "from cal_tools.plotting import (create_constant_overview,\n",
+ " plot_badpix_3d, show_processed_modules,\n",
+ " show_overview)\n",
"from cal_tools.agipdlib import get_gain_setting\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
@@ -118,7 +119,7 @@
"view = Client(profile=cluster_profile)[:]\n",
"view.use_dill()\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
+ "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
"\n",
"gains = np.arange(3)\n",
"\n",
@@ -501,19 +502,17 @@
" 'Noise': noise_g[qm],\n",
" 'ThresholdsDark': thresholds_g[qm],\n",
" 'BadPixelsDark': badpix_g[qm] \n",
- " }\n",
- "\n",
- "if local_output:\n",
- " for qm in offset_g.keys():\n",
- " ofile = \"{}/agipd_offset_store_{}_{}.h5\".format(out_folder,\n",
- " \"{}-{}-{}\".format(*offset_runs.values()), \n",
- " qm)\n",
- " store_file = h5py.File(ofile, \"w\")\n",
- " store_file[\"{}/Offset/0/data\".format(qm)] = offset_g[qm]\n",
- " store_file[\"{}/Noise/0/data\".format(qm)] = noise_g[qm]\n",
- " store_file[\"{}/Threshold/0/data\".format(qm)] = thresholds_g[qm]\n",
- " store_file[\"{}/BadPixels/0/data\".format(qm)] = badpix_g[qm]\n",
- " store_file.close()"
+ " }"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
+ "file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_low, run_med, run_high)"
]
},
{
@@ -534,10 +533,8 @@
"\n",
"for qm in res:\n",
" for const in res[qm]:\n",
- " metadata = ConstantMetaData()\n",
" dconst = getattr(Constants.AGIPD, const)()\n",
" dconst.data = res[qm][const]\n",
- " metadata.calibration_constant = dconst\n",
"\n",
" # Setting conditions\n",
" condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
@@ -557,21 +554,14 @@
" time = mdata.calibration_constant_version.begin_at\n",
" old_mdata[const] = time.isoformat()\n",
" os.makedirs('{}/old/'.format(out_folder), exist_ok=True)\n",
- " save_const_to_h5(mdata, '{}/old/'.format(out_folder))\n",
+ " save_const_to_h5(device,\n",
+ " getattr(Constants.AGIPD, const)(),\n",
+ " condition, data, file_loc, creation_time,\n",
+ " f'{out_folder}/old/')\n",
" else:\n",
" old_mdata[const] = \"Not found\""
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
- "file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_low, run_med, run_high)"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -583,13 +573,12 @@
},
"outputs": [],
"source": [
+ "md = None\n",
"for qm in res:\n",
"\n",
" for const in res[qm]:\n",
- " metadata = ConstantMetaData()\n",
" dconst = getattr(Constants.AGIPD, const)()\n",
" dconst.data = res[qm][const]\n",
- " metadata.calibration_constant = dconst\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
@@ -599,33 +588,18 @@
" detinst = getattr(Detectors, dinstance)\n",
" device = getattr(detinst, qm)\n",
"\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
- " start=creation_time)\n",
- "\n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- "\n",
" if db_output:\n",
- " try:\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " print(f'Const {const} for module {qm} was injected to the calibration DB. '\n",
- " f'Begin at: {metadata.calibration_constant_version.begin_at}')\n",
- " except Exception as e:\n",
- " print(\"Error:\", e)\n",
+ " md = send_to_db(device, dconst, condition, file_loc, \n",
+ " cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
"\n",
" if local_output:\n",
- " save_const_to_h5(metadata, out_folder)\n",
+ " md = save_const_to_h5(device, dconst, condition, dconst.data, file_loc, creation_time, out_folder)\n",
" print(f\"Calibration constant {const} is stored locally.\\n\")\n",
" \n",
- " print(\"Generated constants with conditions:\\n\")\n",
+ " print(\"Constants parameter conditions are:\\n\")\n",
" print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
" f\"• acquisition_rate: {acq_rate}\\n• gain_setting: {gain_setting}\\n\"\n",
- " f\"• creation_time: {creation_time}\\n\")"
+ " f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
},
{
@@ -674,8 +648,7 @@
"source": [
"cell = 3\n",
"gain = 0\n",
- "out_folder = None\n",
- "show_overview(res, cell, gain, out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()))"
+ "show_overview(res, cell, gain, infix=\"{}-{}-{}\".format(*offset_runs.values()))"
]
},
{
@@ -695,7 +668,7 @@
"source": [
"cell = 3\n",
"gain = 1\n",
- "show_overview(res, cell, gain, out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()))"
+ "show_overview(res, cell, gain, infix=\"{}-{}-{}\".format(*offset_runs.values()))"
]
},
{
@@ -715,7 +688,7 @@
"source": [
"cell = 3\n",
"gain = 2\n",
- "show_overview(res, cell, gain, out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()))"
+ "show_overview(res, cell, gain, infix=\"{}-{}-{}\".format(*offset_runs.values()))"
]
},
{
diff --git a/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb b/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
index b0d22ac4ecc2a7a6cb29e3d67858fb2af06f89a3..551f5f98a568931bcc44e659bcf9d141b406e3a0 100644
--- a/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
+++ b/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
@@ -102,7 +102,7 @@
"from cal_tools.tools import (gain_map_files, parse_runs, \n",
" run_prop_seq_from_path, get_notebook_name, \n",
" get_dir_creation_date, save_const_to_h5,\n",
- " get_random_db_interface)\n",
+ " get_random_db_interface, send_to_db)\n",
"from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
@@ -458,11 +458,10 @@
"metadata": {},
"outputs": [],
"source": [
+ "md = None\n",
"for const in res:\n",
- " metadata = ConstantMetaData()\n",
" dconst = getattr(Constants.AGIPD, const)()\n",
" dconst.data = res[const]\n",
- " metadata.calibration_constant = dconst\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
@@ -473,32 +472,20 @@
" # AGIPD_SIV1_AGIPDV11_M001Test\n",
" device = Detectors.AGIPD.AGIPD_SIV1_AGIPDV11_M001Test\n",
"\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
- " start=creation_time)\n",
- "\n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
" if db_output:\n",
- " try:\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " msg = 'Const {} was injected to the calibration DB. Begin at: {}'\n",
- " print(msg.format(const,\n",
- " metadata.calibration_constant_version.begin_at))\n",
- " except Exception as e: \n",
- " if \"has already been take\" in str(e):\n",
- " print(f\"WARN: {const_name} has already been injected with the same parameter conditions\\n\")\n",
- " else:\n",
- " # To prevent having big error message out of the pdf report's page.\n",
- " print(\"\\n\".join(textwrap.wrap(str(e),100)))\n",
+ " md = send_to_db(device, dconst, condition, file_loc, \n",
+ " cal_db_interface, creation_time=creation_time,\n",
+ " timeout=cal_db_timeout)\n",
"\n",
" if local_output:\n",
- " save_const_to_h5(metadata, out_folder)\n",
- " print(\"Calibration constant {} is stored locally.\".format(const))\n"
+ " md = save_const_to_h5(device, dconst, condition,\n",
+ " dconst.data, file_loc, creation_time, out_folder)\n",
+ " print(f\"Calibration constant {const} is stored locally.\\n\")\n",
+ "\n",
+ "print(\"Constants parameter conditions are:\\n\")\n",
+ "print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
+ " f\"• acquisition_rate: {acq_rate}\\n• gain_setting: {gain_setting}\\n\"\n",
+ " f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
},
{
@@ -525,11 +512,11 @@
"source": [
"cell = 3\n",
"gain = 0\n",
- "\n",
+ "# attach module name for plotting.\n",
"res_da = {}\n",
"res_da[module_name] = res\n",
"\n",
- "show_overview(res_da, cell, gain, out_folder=None, infix=\"_\".join(offset_runs.values()))"
+ "show_overview(res_da, cell, gain, infix=\"_\".join(offset_runs.values()))"
]
},
{
@@ -547,7 +534,7 @@
"source": [
"cell = 3\n",
"gain = 1\n",
- "show_overview(res_da, cell, gain, out_folder=None, infix=\"_\".join(offset_runs.values()))"
+ "show_overview(res_da, cell, gain, infix=\"_\".join(offset_runs.values()))"
]
},
{
@@ -565,7 +552,7 @@
"source": [
"cell = 3\n",
"gain = 2\n",
- "show_overview(res_da, cell, gain, out_folder=None, infix=\"_\".join(offset_runs.values()))"
+ "show_overview(res_da, cell, gain, infix=\"_\".join(offset_runs.values()))"
]
},
{
diff --git a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
index 78da0ec2727aa14cc85fdc07844ce0e4207e11a9..dcbd68882f598b217153d22ec0ec6018006eae60 100644
--- a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
+++ b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
@@ -71,36 +71,36 @@
"source": [
"# imports and things that do not usually need to be changed\n",
"from datetime import datetime\n",
+ "import os\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"from collections import OrderedDict\n",
- "import os\n",
+ "\n",
"import h5py\n",
- "import numpy as np\n",
+ "from ipyparallel import Client\n",
+ "from IPython.display import display, Markdown, Latex\n",
"import matplotlib\n",
"matplotlib.use('agg')\n",
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
+ "import numpy as np\n",
+ "import tabulate\n",
"\n",
- "from cal_tools.tools import (map_gain_stages, parse_runs, run_prop_seq_from_path, \n",
- " get_notebook_name, get_dir_creation_date,\n",
- " get_random_db_interface, get_from_db, save_const_to_h5)\n",
- "from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import (show_overview, plot_badpix_3d,\n",
- " create_constant_overview,\n",
+ "from cal_tools.influx import InfluxLogger\n",
+ "from cal_tools.plotting import (create_constant_overview,\n",
+ " plot_badpix_3d, show_overview,\n",
" show_processed_modules)\n",
- "\n",
- "# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
- "from ipyparallel import Client\n",
- "from IPython.display import display, Markdown, Latex\n",
- "import tabulate\n",
- "\n",
+ "from cal_tools.tools import (get_dir_creation_date, get_from_db, \n",
+ " get_notebook_name, get_random_db_interface,\n",
+ " map_gain_stages, parse_runs,\n",
+ " run_prop_seq_from_path, \n",
+ " save_const_to_h5, send_to_db)\n",
+ "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
"view = Client(profile=cluster_profile)[:]\n",
"view.use_dill()\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "\n",
+ "# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
"\n",
"h5path = h5path.format(karabo_id, receiver_id)\n",
"h5path_idx = h5path_idx.format(karabo_id, receiver_id)\n",
@@ -153,7 +153,8 @@
"print(f\"Using DB: {db_output}\")\n",
"print(f\"Input: {in_folder}\")\n",
"print(f\"Output: {out_folder}\")\n",
- "print(f\"Bias voltage: {bias_voltage}V\")"
+ "print(f\"Bias voltage: {bias_voltage}V\")\n",
+ "file_loc = f'proposal:{prop} runs:{[ v for v in offset_runs.values()][0]}'"
]
},
{
@@ -389,7 +390,10 @@
" time = mdata.calibration_constant_version.begin_at\n",
" old_mdata[const] = time.isoformat()\n",
" os.makedirs(f'{out_folder}/old/', exist_ok=True)\n",
- " save_const_to_h5(mdata, f'{out_folder}/old/')\n",
+ " save_const_to_h5(device,\n",
+ " getattr(Constants.DSSC, const)(),\n",
+ " condition, data, file_loc, creation_time,\n",
+ " f'{out_folder}/old/')\n",
" else:\n",
" old_mdata[const] = \"Not found\""
]
@@ -432,48 +436,38 @@
"# Push the same constant two different times.\n",
"# One with the generated pulseID check sum setting for the offline calibration.\n",
"# And another for the online calibration as it doesn't have this pulseID checksum, yet.\n",
+ "md = None\n",
"for dont_use_pulseIds in [True, False]:\n",
" for qm in res.keys():\n",
" detinst = getattr(Detectors, dinstance)\n",
" device = getattr(detinst, qm)\n",
"\n",
" for const in res[qm].keys():\n",
- "\n",
- " metadata = ConstantMetaData()\n",
" dconst = getattr(Constants.DSSC, const)()\n",
" dconst.data = res[qm][const]\n",
- " metadata.calibration_constant = dconst\n",
"\n",
" pidsum = None if dont_use_pulseIds else checksums[qm]\n",
" # set the operating condition\n",
" condition = Conditions.Dark.DSSC(memory_cells=max_cells,\n",
" bias_voltage=bias_voltage,\n",
" pulseid_checksum=pidsum)\n",
- "\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- "\n",
" if db_output:\n",
- " try:\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " except Exception as e:\n",
- " print(\"Error\", e)\n",
+ " md = send_to_db(device, dconst, condition, file_loc, \n",
+ " cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
"\n",
" if local_output:\n",
" # Don't save constant localy two times.\n",
" if dont_use_pulseIds:\n",
- " save_const_to_h5(metadata, out_folder)\n",
+ " md = save_const_to_h5(device, dconst, condition,\n",
+ " dconst.data, file_loc,\n",
+ " creation_time, out_folder)\n",
" print(f\"Calibration constant {const} is stored locally.\\n\")\n",
" \n",
" if not dont_use_pulseIds:\n",
- " print(\"Generated constants with conditions:\\n\")\n",
+ " print(\"Constants parameter conditions are:\\n\")\n",
" print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
- " f\"• pulseid_checksum: {pidsum}\\n• creation_time: {creation_time}\\n\")"
+ " f\"• pulseid_checksum: {pidsum}\\n\"\n",
+ " f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
},
{
diff --git a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
index ac3842e5e253636c0d7410caea790864c81738dd..0c4d704014d69db5f4b346c61b1d180a95d0fbae 100644
--- a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
+++ b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
@@ -21,7 +21,7 @@
"source": [
"cluster_profile = \"noDB\" # ipcluster profile to use\n",
"in_folder = \"/gpfs/exfel/exp/SCS/201930/p900074/raw\" # input folder, required\n",
- "out_folder = '/gpfs/exfel/exp/SCS/201930/p900074/scratch/' # output folder, required\n",
+ "out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/fastccd' # output folder, required\n",
"sequence = 0 # sequence file to use\n",
"run = 351 # which run to read data from, required\n",
"\n",
@@ -75,11 +75,10 @@
"outputs": [],
"source": [
"# Required Packages:\n",
- "\n",
"import copy\n",
"import datetime\n",
- "import os\n",
"import time\n",
+ "import os\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
@@ -90,10 +89,10 @@
"import numpy as np\n",
"from prettytable import PrettyTable\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "from iCalibrationDB.detectors import DetectorTypes\n",
- "from cal_tools.tools import get_dir_creation_date, save_const_to_h5, get_random_db_interface\n",
+ "from cal_tools.tools import get_dir_creation_date, get_random_db_interface, save_const_to_h5\n",
"from cal_tools.enums import BadPixels\n",
+ "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
+ "from iCalibrationDB.detectors import DetectorTypes\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
@@ -1093,12 +1092,10 @@
"dictionary['Offset'] = offsetMap.data\n",
"dictionary['Noise'] = noiseMapCM_2nd.data\n",
"dictionary['BadPixelsDark'] = bad_pixels.data\n",
- "\n",
+ "md = None\n",
"for const in dictionary:\n",
- " metadata = ConstantMetaData()\n",
" dconst = getattr(Constants.CCD(DetectorTypes.fastCCD), const)()\n",
" dconst.data = dictionary[const]\n",
- " metadata.calibration_constant = dconst\n",
" \n",
" condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
" integration_time=integration_time,\n",
@@ -1113,29 +1110,20 @@
" parm.upper_deviation = temp_limits\n",
"\n",
" device = Detectors.fastCCD1\n",
- " metadata.detector_condition = condition\n",
- " \n",
- " # Specifying the a version for this constant:\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " \n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
" \n",
" if db_output:\n",
- " try:\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " print(\"Calibration constant {} is sent to the database.\".format(const))\n",
- " except Exception as e:\n",
- " print(e)\n",
- " \n",
+ " md = send_to_db(device, dconst, condition, file_loc, \n",
+ " cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
+ "\n",
" if local_output:\n",
- " save_const_to_h5(metadata, out_folder)\n",
- " print(\"Calibration constant {} is stored locally.\".format(const))\n",
+ " md = save_const_to_h5(device, dconst, condition, dconst.data, file_loc, creation_time, out_folder)\n",
+ " print(f\"Calibration constant {const} is stored locally.\")\n",
" \n",
- "print(\"Creation time is: {}\".format(creation_time))\n",
- "print(\"Raw data location is: {}\".format(file_loc))"
+ "print(\"Constants parameter conditions are:\\n\")\n",
+ "print(f\"• bias_voltage: {bias_voltage}\\n• integration_time: {integration_time}\\n\"\n",
+ " f\"• temperature: {temperature_k}\\n• gain_setting: {gain_setting}\\n\"\n",
+ " f\"• in_vacuum: {in_vacuum}\\n\"\n",
+ " f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
}
],
diff --git a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
index 4369e67e30389710349e9d28f4b3e54e7d5b571d..931233724f13ce2d06135c6179cc1d4501aedc53 100644
--- a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
@@ -63,38 +63,33 @@
},
"outputs": [],
"source": [
+ "import glob\n",
"import os\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
- "import textwrap\n",
"\n",
+ "import h5py\n",
+ "from h5py import File as h5file\n",
"import matplotlib\n",
"matplotlib.use('agg')\n",
+ "import matplotlib.pyplot as plt\n",
+ "%matplotlib inline\n",
"import numpy as np\n",
- "import h5py\n",
- "from h5py import File as h5file\n",
"\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.tools import get_dir_creation_date, get_random_db_interface\n",
+ "from cal_tools.tools import (get_dir_creation_date, get_random_db_interface,\n",
+ " save_const_to_h5, send_to_db)\n",
"from cal_tools.ana_tools import save_dict_to_hdf5\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
+ "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
"\n",
"from XFELDetAna.util import env\n",
"env.iprofile = cluster_profile\n",
- "\n",
"from XFELDetAna.detectors.jungfrau import readerPSI as jfreaderPSI\n",
"from XFELDetAna.detectors.jungfrau import reader as jfreader\n",
"from XFELDetAna.detectors.jungfrau.jf_chunk_reader import JFChunkReader\n",
"from XFELDetAna.detectors.jungfrau.util import count_n_files, rollout_data, sanitize_data_cellid\n",
- "import glob\n",
- "import matplotlib.pyplot as plt\n",
- "%matplotlib inline\n",
- "\n",
- "from XFELDetAna.plotting.histogram import histPlot\n",
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
- "\n",
- "\n",
- "os.makedirs(out_folder, exist_ok=True)"
+ "from XFELDetAna.plotting.histogram import histPlot"
]
},
{
@@ -108,6 +103,8 @@
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_high, run_med, run_low)\n",
"\n",
+ "os.makedirs(out_folder, exist_ok=True)\n",
+ "\n",
"# TODO \n",
"# this trick is needed until proper mapping is introduced\n",
"if len(db_module)>1:\n",
@@ -450,13 +447,11 @@
"constants = {'Offset': np.moveaxis(offset_map, 0, 1),\n",
" 'Noise': np.moveaxis(noise_map, 0, 1), \n",
" 'BadPixelsDark': np.moveaxis(bad_pixels_map, 0, 1)}\n",
- "\n",
+ "md = None\n",
"for key, const_data in constants.items():\n",
"\n",
- " metadata = ConstantMetaData()\n",
" const = getattr(Constants.jungfrau, key)()\n",
" const.data = const_data\n",
- " metadata.calibration_constant = const\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.jungfrau(memory_cells=memoryCells, bias_voltage=bias_voltage,\n",
@@ -468,51 +463,19 @@
" parm.upper_deviation = time_limits\n",
" \n",
" device = getattr(Detectors, db_module)\n",
- " metadata.detector_condition = condition\n",
"\n",
- " # specify the version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
- " start=creation_time)\n",
- " \n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
" if db_output:\n",
- " try:\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " print(f'Constants {key} is sent to the data base\\n')\n",
- " except Exception as e: \n",
- " if \"has already been take\" in str(e):\n",
- " print(f\"{key} has already been injected with the same parameter conditions\\n\")\n",
- " else:\n",
- " # To prevent having big error message in the pdf report.\n",
- " print(\"\\n\".join(textwrap.wrap(str(e),100)))\n",
- " \n",
- " # save everything to file.\n",
- " # one file - one entrt in the DB\n",
+ " md = send_to_db(device, const, condition, file_loc, \n",
+ " cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
+ "\n",
" if local_output:\n",
- " dpar = {}\n",
- " for parm in metadata.detector_condition.parameters:\n",
- " dpar[parm.name] = {'lower_deviation': parm.lower_deviation,\n",
- " 'upper_deviation': parm.upper_deviation,\n",
- " 'value': parm.value}\n",
- " \n",
- " data_to_store = {}\n",
- " data_to_store['condition'] = dpar\n",
- " data_to_store['db_module'] = db_module\n",
- " data_to_store['constant'] = key\n",
- " data_to_store['data'] = const_data\n",
- " data_to_store['creation_time'] = creation_time\n",
- " data_to_store['dclass'] = 'jungfrau'\n",
- " data_to_store['file_loc'] = file_loc\n",
- " \n",
- " ofile = \"{}/const_{}_{}.h5\".format(out_folder, key, db_module)\n",
- " save_dict_to_hdf5(data_to_store, ofile)\n",
+ " md = save_const_to_h5(device, const, condition, const_data, file_loc, creation_time, out_folder)\n",
" print(f\"Calibration constant {key} is stored locally at {out_folder}.\\n\")\n",
- "print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {memoryCells}\\n\"\n",
+ " \n",
+ "print(\"Constants parameter conditions are:\\n\")\n",
+ "print(f\"• Bias voltage: {bias_voltage}\\n• Memory cells: {memoryCells}\\n\"\n",
" f\"• Integration time: {integration_time}\\n\"\n",
- " f\"• Creation time: {metadata.calibration_constant_version.begin_at}\\n\")"
+ " f\"• Creation time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
},
{
diff --git a/notebooks/LPD/LPDChar_Darks_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
index 53039a126b47cb7368e9fb1cd84881968d19d622..e260e54d23d6bffa9919da3df38ae7ccbf197703 100644
--- a/notebooks/LPD/LPDChar_Darks_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
@@ -77,30 +77,28 @@
"import h5py\n",
"from ipyparallel import Client\n",
"from IPython.display import display, Markdown, Latex\n",
- "import numpy as np\n",
"import matplotlib\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.patches as patches\n",
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
+ "import numpy as np\n",
"import tabulate\n",
- "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
- "from XFELDetAna.plotting.simpleplot import simplePlot\n",
"\n",
- "from iCalibrationDB import (ConstantMetaData, Constants, \n",
- " Conditions, Detectors, \n",
- " Versions)\n",
- "from cal_tools.tools import (map_gain_stages, parse_runs, \n",
- " run_prop_seq_from_path, \n",
- " get_notebook_name, \n",
- " get_dir_creation_date, get_from_db,\n",
- " get_random_db_interface,\n",
- " save_const_to_h5)\n",
- "from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
+ "from cal_tools.influx import InfluxLogger\n",
"from cal_tools.plotting import (show_overview, plot_badpix_3d, \n",
" create_constant_overview,\n",
- " show_processed_modules)"
+ " show_processed_modules)\n",
+ "from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
+ " get_notebook_name, \n",
+ " get_random_db_interface,\n",
+ " map_gain_stages, parse_runs, \n",
+ " run_prop_seq_from_path,\n",
+ " save_const_to_h5, send_to_db)\n",
+ "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
+ "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
+ "from XFELDetAna.plotting.simpleplot import simplePlot"
]
},
{
@@ -338,6 +336,16 @@
"logger.send()"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
+ "file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_low, run_med, run_high)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -372,7 +380,10 @@
" time = mdata.calibration_constant_version.begin_at\n",
" old_mdata[const] = time.isoformat()\n",
" os.makedirs('{}/old/'.format(out_folder), exist_ok=True)\n",
- " save_const_to_h5(mdata, '{}/old/'.format(out_folder))\n",
+ " save_const_to_h5(getattr(detinst, qm),\n",
+ " getattr(Constants.LPD, const)(),\n",
+ " condition, data, file_loc, creation_time,\n",
+ " f'{out_folder}/old/')\n",
" else:\n",
" old_mdata[const] = \"Not found\""
]
@@ -395,16 +406,6 @@
" }"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
- "file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_low, run_med, run_high)"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -412,6 +413,7 @@
"outputs": [],
"source": [
"# Save constants in the calibration DB\n",
+ "md = None\n",
"for cap in capacitor_settings:\n",
" for qm in res[cap]:\n",
" # Do not store empty constants\n",
@@ -420,43 +422,28 @@
" continue\n",
" for const in res[cap][qm]:\n",
"\n",
- " metadata = ConstantMetaData()\n",
" dconst = getattr(Constants.LPD, const)()\n",
" dconst.data = res[cap][qm][const]\n",
- " metadata.calibration_constant = dconst\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.LPD(memory_cells=max_cells,\n",
" bias_voltage=bias_voltage,\n",
" capacitor=cap)\n",
" device = getattr(Detectors.LPD1M1, qm)\n",
- " if device:\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
- " start=creation_time)\n",
- "\n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
"\n",
- " if db_output:\n",
- " try:\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " print(f'Const {const} for module {qm} was injected to the calibration DB. '\n",
- " f'Begin at: {metadata.calibration_constant_version.begin_at}')\n",
- " except Exception as e:\n",
- " print(\"Error:\", e)\n",
+ " if db_output:\n",
+ " md = send_to_db(device, dconst, condition, file_loc, \n",
+ " cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
"\n",
- " if local_output:\n",
- " save_const_to_h5(metadata, out_folder)\n",
- " print(f\"Calibration constant {const} is stored locally.\\n\")\n",
+ " if local_output:\n",
+ " md = save_const_to_h5(device, dconst, condition,\n",
+ " dconst.data, file_loc, creation_time, out_folder)\n",
+ " print(f\"Calibration constant {const} is stored locally.\\n\")\n",
"\n",
- " print(\"Generated constants with conditions:\\n\")\n",
+ " print(\"Constants parameter conditions are:\\n\")\n",
" print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
- " f\"• capacitor: {cap}\\n• creation_time: {creation_time}\\n\")"
+ " f\"• capacitor: {cap}\\n\"\n",
+ " f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
},
{
diff --git a/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb b/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
index d829869675b67d244a4b44f0fd3a2f6634b59eab..ec7c96f0f1b997b5a44e3be5b1cda93881a959a6 100644
--- a/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
@@ -53,32 +53,31 @@
"metadata": {},
"outputs": [],
"source": [
- "from IPython.display import display, Markdown, Latex\n",
"import os\n",
- "import textwrap\n",
+ "import warnings\n",
+ "warnings.filterwarnings('ignore')\n",
"\n",
- "import numpy as np\n",
"import h5py\n",
+ "from IPython.display import display, Markdown, Latex\n",
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
- "import warnings\n",
- "warnings.filterwarnings('ignore')\n",
+ "import numpy as np\n",
"\n",
- "from cal_tools.tools import get_dir_creation_date, save_const_to_h5, get_random_db_interface\n",
- "from iCalibrationDB import (ConstantMetaData, Constants, Conditions, Detectors,\n",
- " Versions)\n",
+ "from cal_tools.tools import (get_dir_creation_date, get_random_db_interface,\n",
+ " save_const_to_h5, send_to_db)\n",
+ "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
- "import XFELDetAna.xfelprofiler as xprof\n",
- "profiler = xprof.Profiler()\n",
- "profiler.disable()\n",
"from XFELDetAna.util import env\n",
"env.iprofile = cluster_profile\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna import xfelpyanatools as xana\n",
+ "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
"from XFELDetAna.plotting.util import prettyPlotting\n",
"prettyPlotting = True\n",
+ "import XFELDetAna.xfelprofiler as xprof\n",
+ "profiler = xprof.Profiler()\n",
+ "profiler.disable()\n",
"from XFELDetAna.xfelreaders import ChunkReader\n",
- "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
"\n",
"h5path = h5path.format(karabo_id, receiver_id)\n",
"h5path_t = h5path_t.format(karabo_id, receiver_id)\n",
@@ -258,21 +257,18 @@
"source": [
"# Save constants to DB\n",
"dclass=\"ePix100\"\n",
- "cal_db_interface = get_random_db_interface(cal_db_interface)\n",
+ "md = None\n",
+ "\n",
"for const_name in constant_maps.keys():\n",
- " metadata = ConstantMetaData()\n",
" det = getattr(Constants, dclass)\n",
" const = getattr(det, const_name)()\n",
" const.data = constant_maps[const_name].data\n",
"\n",
- " metadata.calibration_constant = const\n",
- "\n",
" # set the operating condition\n",
- " dcond = Conditions.Dark\n",
- " condition = getattr(dcond, dclass)(bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " temperature=temperature_k,\n",
- " in_vacuum=in_vacuum)\n",
+ " condition = getattr(Conditions.Dark, dclass)(bias_voltage=bias_voltage,\n",
+ " integration_time=integration_time,\n",
+ " temperature=temperature_k,\n",
+ " in_vacuum=in_vacuum)\n",
"\n",
" for parm in condition.parameters:\n",
" if parm.name == \"Sensor Temperature\":\n",
@@ -280,36 +276,28 @@
" parm.upper_deviation = temp_limits\n",
"\n",
" device = getattr(Detectors, db_module)\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
- " start=creation_time)\n",
- " \n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
"\n",
" if db_output:\n",
- " try:\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " print(f\"Inject {const_name} constants from {metadata.calibration_constant_version.begin_at}\\n\")\n",
- " except Exception as e: \n",
- " if \"has already been take\" in str(e):\n",
- " print(f\"{const_name} has already been injected with the same parameter conditions\\n\")\n",
- " else:\n",
- " # To prevent having big error message in the pdf report.\n",
- " print(\"\\n\".join(textwrap.wrap(str(e),100)))\n",
+ " md = send_to_db(device, const, condition, file_loc, \n",
+ " cal_db_interface, creation_time=creation_time)\n",
"\n",
" if local_output:\n",
- " save_const_to_h5(metadata, out_folder)\n",
- " print(f\"Calibration constant {const_name} is stored locally at {out_folder}.\")\n",
+ " md = save_const_to_h5(device, const, condition,\n",
+ " const.data, file_loc, creation_time, out_folder)\n",
+ " print(f\"Calibration constant {const_name} is stored locally at {out_folder} \\n\")\n",
"\n",
- "print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Integration time: {integration_time}\\n\"\n",
+ "print(\"Constants parameter conditions are:\\n\")\n",
+ "print(f\"• Bias voltage: {bias_voltage}\\n• Integration time: {integration_time}\\n\"\n",
" f\"• Temperature: {temperature_k}\\n• In Vacuum: {in_vacuum}\\n\"\n",
- " f\"• Creation time: {metadata.calibration_constant_version.begin_at}\\n\")"
+ " f\"• Creation time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
diff --git a/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb b/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
index cf8a9d2852dc60d30ff49ecd0a3287c4acad405f..536f8b04b99453c5c95f9c86ce636ed9cae6871a 100644
--- a/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
+++ b/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
@@ -21,7 +21,7 @@
"source": [
"cluster_profile = \"noDB\" # ipcluster profile to use\n",
"in_folder = '/gpfs/exfel/exp/HED/201922/p002550/raw' # input folder, required\n",
- "out_folder = '/gpfs/exfel/exp/HED/201922/p002550/usr/dark/' # output folder, required\n",
+ "out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/epix10' # output folder, required\n",
"sequence = 0 # sequence file to use\n",
"run = 55 # which run to read data from, required\n",
"\n",
@@ -53,34 +53,30 @@
"metadata": {},
"outputs": [],
"source": [
- "import XFELDetAna.xfelprofiler as xprof\n",
- "\n",
- "profiler = xprof.Profiler()\n",
- "profiler.disable()\n",
- "from XFELDetAna.util import env\n",
- "env.iprofile = cluster_profile\n",
- "\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
+ "import os\n",
"\n",
- "from XFELDetAna import xfelpycaltools as xcal\n",
+ "import h5py\n",
+ "import matplotlib.pyplot as plt\n",
+ "%matplotlib inline\n",
+ "import numpy as np\n",
+ "\n",
+ "from cal_tools.tools import (get_dir_creation_date, save_const_to_h5,\n",
+ " get_random_db_interface, send_to_db)\n",
+ "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
+ "from iCalibrationDB.detectors import DetectorTypes\n",
"from XFELDetAna import xfelpyanatools as xana\n",
+ "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna.plotting.util import prettyPlotting\n",
- "\n",
"prettyPlotting = True\n",
- "from XFELDetAna.xfelreaders import ChunkReader\n",
"from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
- "from cal_tools.tools import get_dir_creation_date, save_const_to_h5, get_random_db_interface\n",
- "\n",
- "from iCalibrationDB import (ConstantMetaData, Constants, Conditions, Detectors,\n",
- " Versions)\n",
- "from iCalibrationDB.detectors import DetectorTypes\n",
- "\n",
- "import numpy as np\n",
- "import os\n",
- "import h5py\n",
- "import matplotlib.pyplot as plt\n",
- "%matplotlib inline\n",
+ "from XFELDetAna.util import env\n",
+ "env.iprofile = cluster_profile\n",
+ "import XFELDetAna.xfelprofiler as xprof\n",
+ "profiler = xprof.Profiler()\n",
+ "profiler.disable()\n",
+ "from XFELDetAna.xfelreaders import ChunkReader\n",
"\n",
"h5path = h5path.format(karabo_id, receiver_id)\n",
"h5path_t = h5path_t.format(karabo_id, receiver_id)\n",
@@ -265,15 +261,13 @@
"source": [
"# Save constants to DB\n",
"dclass=\"ePix10K\"\n",
- "cal_db_interface = get_random_db_interface(cal_db_interface)\n",
+ "md = None\n",
+ "\n",
"for const_name in constant_maps.keys():\n",
- " metadata = ConstantMetaData()\n",
" det = getattr(Constants, dclass)\n",
" const = getattr(det, const_name)()\n",
" const.data = constant_maps[const_name].data\n",
"\n",
- " metadata.calibration_constant = const\n",
- "\n",
" # set the operating condition\n",
" dcond = Conditions.Dark\n",
" condition = getattr(dcond, dclass)(bias_voltage=bias_voltage,\n",
@@ -288,29 +282,21 @@
" parm.upper_deviation = temp_limits\n",
"\n",
" device = getattr(Detectors, db_module)\n",
- "\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
- " start=creation_time)\n",
- " \n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
" \n",
" if db_output:\n",
- " try:\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " print(\"Inject {} constants from {}\".format(const_name, \n",
- " metadata.calibration_constant_version.begin_at))\n",
- " except Exception as e:\n",
- " print(e)\n",
- " \n",
+ " md = send_to_db(device, const, const.data, condition, file_loc, \n",
+ " cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
+ "\n",
" if local_output:\n",
- " save_const_to_h5(metadata, out_folder)\n",
- " print(\"Calibration constant {} is stored locally.\".format(const))"
+ " md = save_const_to_h5(device, const, condition,\n",
+ " const.data, file_loc, creation_time, out_folder)\n",
+ " print(f\"Calibration constant {const_name} is stored locally.\")\n",
+ "\n",
+ "print(\"Constants parameter conditions are:\\n\")\n",
+ "print(f\"• bias_voltage: {bias_voltage}\\n• integration_time: {integration_time}\\n\"\n",
+ " f\"• temperature: {temperature_k}\\n• gain_setting: {gain_setting}\\n\"\n",
+ " f\"• in_vacuum: {in_vacuum}\\n\"\n",
+ " f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
}
],
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
index ec232229b6f13f75d2f51be924c9c2d56c22e42d..48c76a03a4151bae898de8827dea9d959dc92479 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
@@ -31,10 +31,10 @@
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # ipcluster profile to use\n",
- "in_folder = \"/gpfs/exfel/exp/SQS/202031/p900166/raw\" # input folder, required\n",
+ "in_folder = \"/gpfs/exfel/exp/SQS/202002/p002714/raw\" # input folder, required\n",
"out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/pnccd' # output folder, required\n",
"sequence = 0 # sequence file to use\n",
- "run = 39 # which run to read data from, required\n",
+ "run = 281 # which run to read data from, required\n",
"\n",
"db_module = \"pnCCD_M205_M206\"\n",
"karabo_da = ['PNCCD01'] # data aggregators\n",
@@ -94,7 +94,10 @@
"from prettytable import PrettyTable\n",
"from IPython.display import display, Markdown\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
+ "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.tools import (get_dir_creation_date, save_const_to_h5,\n",
+ " get_random_db_interface, send_to_db)\n",
+ "from iCalibrationDB import (Constants, Conditions, Detectors, Versions)\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
"import XFELDetAna.xfelprofiler as xprof\n",
"profiler = xprof.Profiler()\n",
@@ -106,9 +109,7 @@
"from XFELDetAna.plotting.util import prettyPlotting\n",
"prettyPlotting=True\n",
"from XFELDetAna.xfelreaders import ChunkReader\n",
- "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
- "from cal_tools.tools import get_dir_creation_date, save_const_to_h5, get_random_db_interface\n",
- "from cal_tools.enums import BadPixels"
+ "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5"
]
},
{
@@ -998,13 +999,11 @@
" 'Noise': noiseMapCM_2nd,\n",
" 'BadPixelsDark': bad_pixels\n",
"}\n",
- "\n",
+ "md = None\n",
"for const_name in constant_maps.keys():\n",
- " metadata = ConstantMetaData()\n",
" det = Constants.CCD(DetectorTypes.pnCCD)\n",
" const = getattr(det, const_name)()\n",
" const.data = constant_maps[const_name].data\n",
- " metadata.calibration_constant = const\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
@@ -1020,51 +1019,19 @@
" parm.upper_deviation = temp_limits\n",
"\n",
" device = getattr(Detectors, db_module)\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
- " start=creation_time)\n",
- "\n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
"\n",
" if db_output:\n",
- " try:\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " print(f\"Inject {const_name} constants from {metadata.calibration_constant_version.begin_at}\\n\")\n",
- " except Exception as e: \n",
- " if \"has already been take\" in str(e):\n",
- " print(f\"WARN: {const_name} has already been injected with the same parameter conditions\\n\")\n",
- " else:\n",
- " # To prevent having big error message out of the pdf report's page.\n",
- " print(\"\\n\".join(textwrap.wrap(str(e),100)))\n",
- "\n",
+ " md = send_to_db(device, const, condition, file_loc, \n",
+ " cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
" if local_output:\n",
- " save_const_to_h5(metadata, out_folder)\n",
- " print(f\"Calibration constant {const.name} is stored to {out_folder}.\\n\")\n",
+ " md = save_const_to_h5(device, const, condition, const.data, file_loc, creation_time, out_folder)\n",
+ " print(f\"Calibration constant {const_name} is stored to {out_folder}.\\n\")\n",
"\n",
- "print(\"Generated constants with conditions:\\n\")\n",
+ "print(\"Constants parameter conditions are:\\n\")\n",
"print(f\"• bias_voltage: {bias_voltage}\\n• integration_time: {integration_time}\\n\"\n",
" f\"• gain_setting: {gain}\\n• temperature: {fix_temperature}\\n\"\n",
- " f\"• creation_time: {creation_time}\\n\")"
+ " f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {