diff --git a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
index 10da19ed8966e13ca35b95e3746f1fe75413ed15..d0130889643859dc904448f39d0346ae71636ff7 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
@@ -31,16 +31,19 @@
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # ipcluster profile to use\n",
- "in_folder = \"/gpfs/exfel/exp/SQS/201930/p900075/raw\" # input folder, required\n",
+ "in_folder = \"/gpfs/exfel/exp/SQS/202031/p900166/raw\" # input folder, required\n",
"out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/pnccd' # output folder, required\n",
"sequence = 0 # sequence file to use\n",
- "run = 364 # which run to read data from, required\n",
+ "run = 39 # which run to read data from, required\n",
"\n",
+ "db_module = \"pnCCD_M205_M206\"\n",
"karabo_da = ['PNCCD01'] # data aggregators\n",
+ "karabo_da_control = \"PNCCD02\" # file inset for control data\n",
"karabo_id = \"SQS_NQS_PNCCD1MP\" # karabo prefix of PNCCD devices\n",
"receiver_id = \"PNCCD_FMT-0\" # inset for receiver devices\n",
"path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
"h5path = '/INSTRUMENT/{}/CAL/{}:output/data/image/' # path in the HDF5 file the data is at\n",
+ "h5path_ctrl = '/CONTROL/{}/CTRL/TCTRL'\n",
"\n",
"# for database time derivation:\n",
"use_dir_creation_date = True # use dir creation date as data production reference date\n",
@@ -52,9 +55,9 @@
"\n",
"number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
"chunkSize = 100 # number of images to read per chunk\n",
- "fix_temperature = 233. # fix temperature in K, set to 0. to use value from slow data\n",
- "gain = 1 # the detector's gain setting, only 1 and 64 is available.\n",
- "bias_voltage = 300 # detector's bias voltage\n",
+ "fix_temperature = 0. # fix temperature in K, set to 0. to use value from slow data\n",
+ "gain = 0. # the detector's gain setting, It is later read from file and this value is overwritten\n",
+ "bias_voltage = 0. # the detector's bias voltage. set to 0. to use value from slow data.\n",
"integration_time = 70 # detector's integration time\n",
"commonModeAxis = 0 # axis along which common mode will be calculated (0: along rows, 1: along columns)\n",
"commonModeBlockSize = [512, 512] # size of the detector in pixels for common mode calculations\n",
@@ -84,6 +87,7 @@
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
+ "import h5py\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
@@ -107,17 +111,6 @@
"from cal_tools.enums import BadPixels"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Output Folder Creation:\n",
- "if not os.path.exists(out_folder):\n",
- " os.makedirs(out_folder)"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -143,7 +136,7 @@
"outputs": [],
"source": [
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
- "file_loc = 'Proposal: {}, Run: {}'.format(proposal, run)\n",
+ "file_loc = f'Proposal: {proposal}, Run: {run}'\n",
"print(\"File Location:\", file_loc)"
]
},
@@ -171,6 +164,9 @@
"filename = fp_path.format(sequence)\n",
"h5path = h5path.format(karabo_id, receiver_id)\n",
"\n",
+ "# Output Folder Creation:\n",
+ "os.makedirs(out_folder, exist_ok=True)\n",
+ "\n",
"# Run's creation time:\n",
"if creation_time:\n",
" try:\n",
@@ -196,6 +192,33 @@
"print(\"Run number: {}\".format(run))"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# extract slow data\n",
+ "if karabo_da_control:\n",
+ " ctrl_fname = os.path.join(ped_dir, path_template.format(run, karabo_da_control)).format(sequence)\n",
+ " ctrl_path = h5path_ctrl.format(karabo_id)\n",
+ " mdl_ctrl_path = f\"/CONTROL/{karabo_id}/MDL/\"\n",
+ " try:\n",
+ " with h5py.File(ctrl_fname, \"r\") as f:\n",
+ " if bias_voltage == 0.:\n",
+ " print(\"bias voltage control h5path:\", os.path.join(mdl_ctrl_path, \"DAQ_MPOD/u0voltage/value\"))\n",
+ " bias_voltage = abs(f[os.path.join(mdl_ctrl_path, \"DAQ_MPOD/u0voltage/value\")][0])\n",
+ " gain = f[os.path.join(mdl_ctrl_path, \"DAQ_GAIN/pNCCDGain/value\")][0]\n",
+ " if fix_temperature == 0.:\n",
+ " fix_temperature = f[os.path.join(ctrl_path, \"inputA/krdg/value\")][0]\n",
+ " except KeyError:\n",
+ " print(\"Error !!! during extracting slow data\")\n",
+ " traceback.print_exc(limit=1)\n",
+ " print(\"bias voltage control h5path:\", os.path.join(mdl_ctrl_path, \"DAQ_MPOD/u0voltage/value\"))\n",
+ " print(\"gain control h5path:\", os.path.join(mdl_ctrl_path, \"DAQ_GAIN/pNCCDGain/value\"))\n",
+ " print(\"fix_temperature control h5path:\", os.path.join(ctrl_path, \"inputA/krdg/value\"))"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -208,7 +231,6 @@
"outputs": [],
"source": [
"# Reading Parameters such as Detector Bias, Gain, etc. from the Data:\n",
- "\n",
"memoryCells = 1 # pnCCD has 1 memory cell\n",
"sensorSize = [pixels_x, pixels_y]\n",
"blockSize = [sensorSize[0]//2, sensorSize[1]//2]# sensor area will be analysed according to blocksize\n",
@@ -225,21 +247,11 @@
"outputs": [],
"source": [
"# Printing the Parameters Read from the Data File:\n",
- "\n",
"display(Markdown('### Detector Parameters'))\n",
- "print(\"Bias voltage is {} V.\".format(bias_voltage))\n",
- "print(\"Detector gain is set to {}.\".format(gain))\n",
- "print(\"Detector integration time is set to {} ms\".format(integration_time)) \n",
- "\n",
- "if fix_temperature != 0.:\n",
- " print(f\"Using a fixed temperature of {fix_temperature} K\")\n",
- " temperature_k = fix_temperature\n",
- "else:\n",
- " print(\"Temperature is not fixed.\")\n",
- " #TODO: remove this line after properly saving the temperature in control data.\n",
- " temperature_k = 233.\n",
- " print(f\"Using a fixed temperature of {fix_temperature} K\")\n",
- " \n",
+ "print(f\"Bias voltage is {bias_voltage} V.\")\n",
+ "print(f\"Detector gain is set to {gain}.\")\n",
+ "print(f\"Detector integration time is set to {integration_time} ms\") \n",
+ "print(f\"Using a fixed temperature of {fix_temperature} K\")\n",
"print(\"Number of dark images to analyze:\", nImages) "
]
},
@@ -993,18 +1005,22 @@
" det = Constants.CCD(DetectorTypes.pnCCD)\n",
" const = getattr(det, const_name)()\n",
" const.data = constant_maps[const_name].data\n",
- "\n",
" metadata.calibration_constant = const\n",
"\n",
" # set the operating condition\n",
" condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
" integration_time=integration_time,\n",
" gain_setting=gain,\n",
- " temperature=temperature_k,\n",
+ " temperature=fix_temperature,\n",
" pixels_x=pixels_x,\n",
" pixels_y=pixels_y)\n",
"\n",
- " device = Detectors.PnCCD1\n",
+ " for parm in condition.parameters:\n",
+ " if parm.name == \"Sensor Temperature\":\n",
+ " parm.lower_deviation = temp_limits\n",
+ " parm.upper_deviation = temp_limits\n",
+ "\n",
+ " device = getattr(Detectors, db_module)\n",
" metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
@@ -1018,10 +1034,14 @@
"\n",
" if db_output:\n",
" try:\n",
- " metadata.send(cal_db_interface)\n",
- " print(f\"Inject {const_name} constants from {metadata.calibration_constant_version.begin_at}\")\n",
- " except Exception as e:\n",
- " print(e)\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
+ " print(f\"Inject {const_name} constants from {metadata.calibration_constant_version.begin_at}\\n\")\n",
+ " except Exception as e: \n",
+ " if \"has already been take\" in str(e):\n",
+ " print(f\"WARN: {const_name} has already been injected with the same parameter conditions\\n\")\n",
+ " else:\n",
+ " # To prevent having big error message out of the pdf report's page.\n",
+ " print(\"\\n\".join(textwrap.wrap(str(e),100)))\n",
"\n",
" if local_output:\n",
" save_const_to_h5(metadata, out_folder)\n",
@@ -1029,10 +1049,17 @@
"\n",
"print(\"Generated constants with conditions:\\n\")\n",
"print(f\"• bias_voltage: {bias_voltage}\\n• integration_time: {integration_time}\\n\"\n",
- " f\"• gain_setting: {gain}\\n• temperature: {temperature_k}\\n\"\n",
+ " f\"• gain_setting: {gain}\\n• temperature: {fix_temperature}\\n\"\n",
" f\"• creation_time: {creation_time}\\n\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index a41a37a6a61bf9716311114bf3bb961b1494edd1..b1484e339ed14465b720bee8c84e7a500fd416a5 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -22,17 +22,20 @@
},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/SQS/201930/p900075/raw\" # input folder\n",
+ "in_folder = \"/gpfs/exfel/exp/SQS/202031/p900166/raw\" # input folder\n",
"out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/pnccd' # output folder\n",
- "run = 365 # which run to read data from\n",
- "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
+ "run = 39 # which run to read data from\n",
+ "sequences = [0] # sequences to correct, set to -1 for all, range allowed\n",
"\n",
+ "db_module = \"pnCCD_M205_M206\"\n",
"karabo_da = 'PNCCD01' # data aggregators\n",
+ "karabo_da_control = \"PNCCD02\" # file inset for control data\n",
"karabo_id = \"SQS_NQS_PNCCD1MP\" # karabo prefix of PNCCD devices\n",
"receiver_id = \"PNCCD_FMT-0\" # inset for receiver devices\n",
- "path_template = 'RAW-R{:04d}-PNCCD01-S{{:05d}}.h5' # the template to use to access data\n",
+ "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
"path_template_seqs = \"{}/r{:04d}/*PNCCD01-S*.h5\"\n",
"h5path = '/INSTRUMENT/{}/CAL/{}:output/data/' # path to data in the HDF5 file \n",
+ "h5path_ctrl = '/CONTROL/{}/CTRL/TCTRL'\n",
"\n",
"overwrite = True # keep this as True to not overwrite the output \n",
"use_dir_creation_date = True # required to obtain creation time of the run\n",
@@ -51,9 +54,9 @@
"seq_num = 0 # sequence number for which the last plot at the end of the notebook is plotted\n",
"\n",
"# pnCCD parameters:\n",
- "fix_temperature = 233.\n",
- "gain = 1\n",
- "bias_voltage = 300\n",
+ "fix_temperature = 0. # fix temperature in K, set to 0. to use value from slow data.\n",
+ "gain = 0. # the detector's gain setting, It is later read from file and this value is overwritten\n",
+ "bias_voltage = 0. # the detector's bias voltage. set to 0. to use value from slow data.\n",
"integration_time = 70\n",
"photon_energy = 1.6 # Al fluorescence in keV\n",
"\n",
@@ -104,6 +107,7 @@
"import time\n",
"import copy\n",
"import os\n",
+ "import traceback\n",
"import glob\n",
"import datetime\n",
"from datetime import timedelta\n",
@@ -134,56 +138,6 @@
" sequences = None"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Each xcal.HistogramCalculator requires a total number of bins and a binning range. We define these using a \n",
- "# dictionary:\n",
- "\n",
- "# For all xcal histograms:\n",
- "if gain == 1:\n",
- " Hist_Bin_Dict = {\n",
- " \"bins\": 70000, # number of bins \n",
- " \"bin_range\": [0, 70000]\n",
- " }\n",
- "\n",
- " # For the numpy histograms on the last cell of the notebook:\n",
- " Event_Bin_Dict = {\n",
- " \"event_bins\": 1000, # number of bins \n",
- " \"b_range\": [0, 50000] # bin range \n",
- " }\n",
- " \n",
- "elif gain == 64:\n",
- " # For all xcal histograms:\n",
- " Hist_Bin_Dict = {\n",
- " \"bins\": 25000, # number of bins \n",
- " \"bin_range\": [0, 25000] \n",
- " }\n",
- " # For the numpy histograms on the last cell of the notebook:\n",
- " Event_Bin_Dict = {\n",
- " \"event_bins\": 1000, # number of bins \n",
- " \"b_range\": [0, 3000] # bin range \n",
- " }\n",
- " \n",
- "bins = Hist_Bin_Dict[\"bins\"]\n",
- "bin_range = Hist_Bin_Dict[\"bin_range\"]\n",
- "event_bins = Event_Bin_Dict[\"event_bins\"]\n",
- "b_range = Event_Bin_Dict[\"b_range\"]\n",
- "\n",
- "# On the singles spectrum (uploaded in the middle of this notebook), the ADU values correspoding to the boundaries\n",
- "# of the first peak region are used as cti_limit_low and cti_limit_high:\n",
- "\n",
- "if gain == 1:\n",
- " cti_limit_low = 3000 # lower limit of cti\n",
- " cti_limit_high = 10000 # higher limit of cti\n",
- "elif gain == 64:\n",
- " cti_limit_low = 50\n",
- " cti_limit_high = 170"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -224,9 +178,15 @@
" creation_time = get_dir_creation_date(in_folder, run)\n",
"\n",
"\n",
- "print(f\"Creation time: {creation_time}\")\n",
- " \n",
- "\n",
+ "print(f\"Creation time: {creation_time}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"# Reading all sequences of the run:\n",
"file_list = []\n",
"total_sequences = 0\n",
@@ -249,6 +209,48 @@
"print(f\"This run has a total number of {total_sequences} sequences.\\n\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# extract slow data\n",
+ "if karabo_da_control:\n",
+ " ctrl_fname = os.path.join(ped_dir, path_template.format(run, karabo_da_control)).format(sequences[0])\n",
+ " ctrl_path = h5path_ctrl.format(karabo_id)\n",
+ " mdl_ctrl_path = f\"/CONTROL/{karabo_id}/MDL/\"\n",
+ " try:\n",
+ " with h5py.File(ctrl_fname, \"r\") as f:\n",
+ " if bias_voltage == 0.:\n",
+ " bias_voltage = abs(f[os.path.join(mdl_ctrl_path, \"DAQ_MPOD/u0voltage/value\")][0])\n",
+ " gain = f[os.path.join(mdl_ctrl_path, \"DAQ_GAIN/pNCCDGain/value\")][0]\n",
+ " if fix_temperature == 0.:\n",
+ " fix_temperature = f[os.path.join(ctrl_path, \"inputA/krdg/value\")][0]\n",
+ " except KeyError:\n",
+ " print(\"Error !!! during extracting slow data\")\n",
+ " traceback.print_exc(limit=1)\n",
+ " print(\"Control file name:\", ctrl_fname)\n",
+ " print(\"bias voltage control h5path:\", os.path.join(mdl_ctrl_path, \"DAQ_MPOD/u0voltage/value\"))\n",
+ " print(\"gain control h5path:\", os.path.join(mdl_ctrl_path, \"DAQ_GAIN/pNCCDGain/value\"))\n",
+ " print(\"fix_temperature control h5path:\", os.path.join(ctrl_path, \"inputA/krdg/value\"))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Printing the Parameters Read from the Data File:\n",
+ "\n",
+ "display(Markdown('### Detector Parameters'))\n",
+ "print(\"Bias voltage is {} V.\".format(bias_voltage))\n",
+ "print(\"Detector gain is set to {}.\".format(gain))\n",
+ "print(\"Detector integration time is set to {} ms\".format(integration_time))\n",
+ "print(f\"Using a fixed temperature of {fix_temperature} K\")"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -297,6 +299,58 @@
" raise AttributeError(\"Output path exists! Exiting\") "
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Each xcal.HistogramCalculator requires a total number of bins and a binning range. We define these using a \n",
+ "# dictionary:\n",
+ "\n",
+ "# For all xcal histograms:\n",
+ "if gain == 1:\n",
+ " Hist_Bin_Dict = {\n",
+ " \"bins\": 70000, # number of bins \n",
+ " \"bin_range\": [0, 70000]\n",
+ " }\n",
+ "\n",
+ " # For the numpy histograms on the last cell of the notebook:\n",
+ " Event_Bin_Dict = {\n",
+ " \"event_bins\": 1000, # number of bins \n",
+ " \"b_range\": [0, 50000] # bin range \n",
+ " }\n",
+ " \n",
+ "#TODO: make it more adaptive for more than only 2 gains [below was for gain==64 only]\n",
+ "else:\n",
+ " # For all xcal histograms:\n",
+ " Hist_Bin_Dict = {\n",
+ " \"bins\": 25000, # number of bins \n",
+ " \"bin_range\": [0, 25000] \n",
+ " }\n",
+ " # For the numpy histograms on the last cell of the notebook:\n",
+ " Event_Bin_Dict = {\n",
+ " \"event_bins\": 1000, # number of bins \n",
+ " \"b_range\": [0, 3000] # bin range \n",
+ " }\n",
+ " \n",
+ "bins = Hist_Bin_Dict[\"bins\"]\n",
+ "bin_range = Hist_Bin_Dict[\"bin_range\"]\n",
+ "event_bins = Event_Bin_Dict[\"event_bins\"]\n",
+ "b_range = Event_Bin_Dict[\"b_range\"]\n",
+ "\n",
+ "# On the singles spectrum (uploaded in the middle of this notebook), the ADU values correspoding to the boundaries\n",
+ "# of the first peak region are used as cti_limit_low and cti_limit_high:\n",
+ "\n",
+ "if gain == 1:\n",
+ " cti_limit_low = 3000 # lower limit of cti\n",
+ " cti_limit_high = 10000 # higher limit of cti\n",
+ "#TODO: make it more adaptive for more than only 2 gains [below was for gain==64 only\n",
+ "else:\n",
+ " cti_limit_low = 50\n",
+ " cti_limit_high = 170"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -337,7 +391,7 @@
" \n",
" for const in constants.keys():\n",
" constants[const], when[const] = \\\n",
- " get_constant_from_db_and_time(Detectors.PnCCD1,\n",
+ " get_constant_from_db_and_time(getattr(Detectors, db_module),\n",
" getattr(Constants.CCD(DetectorTypes.pnCCD), const)(),\n",
" condition,\n",
" np.zeros((pixels_x, pixels_y, 1)),\n",
@@ -408,7 +462,7 @@
" photon_energy=photon_energy)\n",
"\n",
" constants[\"RelativeGain\"], relgain_time = \\\n",
- " get_constant_from_db_and_time(Detectors.PnCCD1,\n",
+ " get_constant_from_db_and_time(getattr(Detectors, db_module),\n",
" Constants.CCD(DetectorTypes.pnCCD).RelativeGain(),\n",
" condition,\n",
" np.zeros((pixels_x, pixels_y)),\n",
@@ -806,7 +860,8 @@
"\n",
"if gain == 1:\n",
" x_range = (0, 30000)\n",
- "elif gain == 64:\n",
+ "#TODO: make it more adaptive for more than only 2 gains [below was for gain==64 only\n",
+ "else:\n",
" x_range = (0, 1000)"
]
},
@@ -1271,6 +1326,13 @@
" l = ax.legend()"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
{
"cell_type": "code",
"execution_count": null,