From b2341f5d63bac13d1c51c21e0fd87b3cbdf7a8c4 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Wed, 10 Nov 2021 09:42:52 +0100
Subject: [PATCH] remove unneeded parameters and alg.
---
.../pnCCD/Characterize_pnCCD_Dark_NBC.ipynb | 98 +++++++++----------
notebooks/pnCCD/Correct_pnCCD_NBC.ipynb | 59 +++++------
2 files changed, 71 insertions(+), 86 deletions(-)
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
index 1cbdcd490..a1e303b74 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
@@ -31,7 +31,6 @@
"\n",
"# Data files parameters.\n",
"karabo_da = ['PNCCD01'] # data aggregators\n",
- "karabo_da_control = \"PNCCD02\" # file inset for control data\n",
"karabo_id = \"SQS_NQS_PNCCD1MP\" # karabo prefix of PNCCD devices\n",
"receiver_id = \"PNCCD_FMT-0\" # inset for receiver devices\n",
"path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
@@ -46,6 +45,7 @@
"creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. 2019-07-04 11:02:41.00\n",
"\n",
"# Parameters used for injecting calibration constants.\n",
+ "set_parameters_manually = False # Boolean for setting parameter values manually instead of reading control values.\n",
"fix_temperature_top = 0. # fix temperature of top pnCCD sensor in K. Set to 0, to use the value from slow data\n",
"fix_temperature_bot = 0. # fix temperature of bottom pnCCD sensor in K. Set to 0, to use the value from slow data\n",
"temp_limits = 5 # temperature limits in which calibration parameters are considered equal\n",
@@ -87,7 +87,6 @@
"from extra_data import RunDirectory\n",
"\n",
"%matplotlib inline\n",
- "import XFELDetAna.xfelprofiler as xprof\n",
"from cal_tools import pnccdlib\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
@@ -104,9 +103,6 @@
"from IPython.display import Markdown, display\n",
"from prettytable import PrettyTable\n",
"\n",
- "profiler = xprof.Profiler()\n",
- "profiler.disable()\n",
- "\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna import xfelpycaltools as xcal"
]
@@ -189,7 +185,7 @@
"run_dc = RunDirectory(f\"{in_folder}/r{run:04d}\")\n",
"\n",
"# extract control data\n",
- "if karabo_da_control:\n",
+ "if not set_parameters_manually:\n",
" ctrl_data = pnccdlib.PnccdCtrl(run_dc, karabo_id)\n",
" \n",
" if bias_voltage == 0:\n",
@@ -219,9 +215,6 @@
"outputs": [],
"source": [
"# Reading Parameters such as Detector Bias, Gain, etc. from the Data:\n",
- "profile = False\n",
- "run_parallel = False # for parallel computation\n",
- "memoryCells = 1 # pnCCD has 1 memory cell\n",
"sensorSize = [pixels_x, pixels_y]\n",
"# sensor area will be analysed according to blocksize\n",
"blockSize = [sensorSize[0]//2, sensorSize[1]//2]\n",
@@ -232,7 +225,7 @@
"n_trains = data_dc[pixel_data].shape[0]\n",
"\n",
"if max_trains != 0:\n",
- " n_trains = max_trains\n",
+ " n_trains = min(n_trains, max_trains)\n",
"if n_trains < min_trains:\n",
" raise ValueError(\n",
" f\"Files {data_dc.files} consists of less than\"\n",
@@ -387,7 +380,7 @@
" sensorSize,\n",
" commonModeBlockSize,\n",
" commonModeAxis,\n",
- " parallel=run_parallel,\n",
+ " parallel=False,\n",
" dType=np.float32, stride=1,\n",
" noiseMap=noiseMap[..., np.newaxis], # np.float32\n",
" minFrac=0,\n",
@@ -405,11 +398,11 @@
"# negative bin to ensure the offset and common mode corrections actually move the signal to zero:\n",
"\n",
"# For offset corrected data:\n",
- "histCalCorrected = xcal.HistogramCalculator(sensorSize, bins=bins, range=bin_range, memoryCells=memoryCells,\n",
- " parallel=run_parallel, gains=None, blockSize=blockSize)\n",
+ "histCalCorrected = xcal.HistogramCalculator(sensorSize, bins=bins, range=bin_range, memoryCells=1,\n",
+ " parallel=False, gains=None, blockSize=blockSize)\n",
"# For common mode corrected data:\n",
- "histCalCMCorrected = xcal.HistogramCalculator(sensorSize, bins=bins, range=bin_range, memoryCells=memoryCells,\n",
- " parallel=run_parallel, gains=None, blockSize=blockSize)"
+ "histCalCMCorrected = xcal.HistogramCalculator(sensorSize, bins=bins, range=bin_range, memoryCells=1,\n",
+ " parallel=False, gains=None, blockSize=blockSize)"
]
},
{
@@ -658,7 +651,7 @@
" sensorSize,\n",
" commonModeBlockSize,\n",
" commonModeAxis,\n",
- " parallel=run_parallel,\n",
+ " parallel=False,\n",
" dType=np.float32, stride=1,\n",
" noiseMap=noiseMapCM[..., np.newaxis], minFrac=0,\n",
")"
@@ -674,30 +667,26 @@
"def correct_image(wid, idx, d):\n",
" \n",
" d[bad_pixels != 0] = np.nan # masking data for bad pixels and equating the values to np.nan\n",
- " d_off1 = d - offsetMap\n",
- "\n",
- " d_off1 = np.squeeze(cmCorrection.correct(\n",
- " d_off1[..., np.newaxis],\n",
- " cellTable=np.zeros((1,), dtype=np.float32)))\n",
- "\n",
- " d[d_off1 > event_threshold] = np.nan # discarding events caused by cosmic rays\n",
+ " d_off = d - offsetMap\n",
+ " d_off_cm = np.squeeze(cmCorrection.correct(\n",
+ " d_off[..., np.newaxis], cellTable=np.zeros((1,), dtype=np.float32)))\n",
"\n",
- " d_off2 = d - offsetMap # Offset correction\n",
- " offset_corr_data2[idx, ...] = d_off2 # Will be used for plotting\n",
- " histCalCorrected.fill(d_off2[..., np.newaxis])\n",
+ " # discarding events caused by cosmic rays\n",
+ " d_off[d_off_cm > event_threshold] = np.nan\n",
+ " d_off_cm[d_off_cm > event_threshold] = np.nan \n",
"\n",
- " d_off2 = cmCorrection.correct(\n",
- " d_off2,\n",
- " cellTable=np.zeros((1,), dtype=np.float32))\n",
- "\n",
- " histCalCMCorrected.fill(d_off2)\n",
- " corr_data[idx, ...] = np.squeeze(d_off2)\n",
+ " histCalCorrected.fill(d_off[..., np.newaxis])\n",
"\n",
+ " offset_corr_data[idx, ...] = d_off # Will be used for plotting\n",
+ " \n",
+ " corr_data[idx, ...] = np.squeeze(d_off_cm)\n",
+ " histCalCorrected.fill(d_off)\n",
+ " histCalCMCorrected.fill(d_off_cm)\n",
"context = psh.context.ThreadContext(num_workers=10)\n",
"\n",
"corr_data = context.alloc(\n",
" shape=(n_trains, sensorSize[0], sensorSize[1]), dtype=np.float32)\n",
- "offset_corr_data2 = context.alloc(\n",
+ "offset_corr_data = context.alloc(\n",
" shape=(n_trains, sensorSize[0], sensorSize[1]), dtype=np.float32)\n",
"\n",
"context.map(correct_image, data.copy())\n",
@@ -758,11 +747,11 @@
"t0 = PrettyTable()\n",
"t0.title = \"Comparison of the Second Round of Corrections - Bad Pixels Excluded\"\n",
"t0.field_names = [\"Dark Pedestal After Offset Correction\", \"Dark Pedestal After Offset and Common Mode Corrections\"]\n",
- "t0.add_row([\"Mean: {:0.3f} ADU\".format(np.nanmean(offset_corr_data2)), \"Mean: {:0.3f} ADU\"\n",
+ "t0.add_row([\"Mean: {:0.3f} ADU\".format(np.nanmean(offset_corr_data)), \"Mean: {:0.3f} ADU\"\n",
" .format(np.nanmean(corr_data))])\n",
- "t0.add_row([\"Median: {:0.3f} ADU\".format(np.nanmedian(offset_corr_data2)), \"Median: {:0.3f} ADU\"\n",
+ "t0.add_row([\"Median: {:0.3f} ADU\".format(np.nanmedian(offset_corr_data)), \"Median: {:0.3f} ADU\"\n",
" .format(np.nanmedian(corr_data))])\n",
- "t0.add_row([\"Standard Deviation: {:0.3f} ADU\".format(np.nanstd(offset_corr_data2)),\n",
+ "t0.add_row([\"Standard Deviation: {:0.3f} ADU\".format(np.nanstd(offset_corr_data)),\n",
" \"Standard Deviation: {:0.3f} ADU\".format(np.nanstd(corr_data))])\n",
"print(t0,'\\n')\n",
"step_timer.done_step(\"Plotting time\")"
@@ -873,28 +862,29 @@
" 'BadPixelsDark': bad_pixels[..., np.newaxis],\n",
"}\n",
"md = None\n",
+ "\n",
+ "det = Constants.CCD(DetectorTypes.pnCCD)\n",
+ "\n",
+ "# set the operating condition\n",
+ "condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
+ " integration_time=integration_time,\n",
+ " gain_setting=gain,\n",
+ " temperature=fix_temperature_top,\n",
+ " pixels_x=pixels_x,\n",
+ " pixels_y=pixels_y)\n",
+ "for parm in condition.parameters:\n",
+ " if parm.name == \"Sensor Temperature\":\n",
+ " parm.lower_deviation = temp_limits\n",
+ " parm.upper_deviation = temp_limits\n",
+ "\n",
+ "db_module = get_pdu_from_db(karabo_id, karabo_da, getattr(det, 'Offset')(),\n",
+ " condition, cal_db_interface,\n",
+ " snapshot_at=creation_time)[0]\n",
+ "\n",
"for const_name in constant_maps.keys():\n",
- " det = Constants.CCD(DetectorTypes.pnCCD)\n",
" const = getattr(det, const_name)()\n",
" const.data = constant_maps[const_name].data\n",
"\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " gain_setting=gain,\n",
- " temperature=fix_temperature_top,\n",
- " pixels_x=pixels_x,\n",
- " pixels_y=pixels_y)\n",
- "\n",
- " for parm in condition.parameters:\n",
- " if parm.name == \"Sensor Temperature\":\n",
- " parm.lower_deviation = temp_limits\n",
- " parm.upper_deviation = temp_limits\n",
- "\n",
- " db_module = get_pdu_from_db(karabo_id, karabo_da, const,\n",
- " condition, cal_db_interface,\n",
- " snapshot_at=creation_time)[0]\n",
- "\n",
" if db_output:\n",
" md = send_to_db(db_module, karabo_id, const, condition,\n",
" file_loc=file_loc, report_path=report,\n",
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index 6e83b31d2..181eb9825 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -24,7 +24,6 @@
"sequences_per_node = 1 # number of sequences running on the same slurm node.\n",
"\n",
"karabo_da = 'PNCCD01' # data aggregators\n",
- "karabo_da_control = \"PNCCD02\" # file inset for control data\n",
"karabo_id = \"SQS_NQS_PNCCD1MP\" # karabo prefix of PNCCD devices\n",
"receiver_id = \"PNCCD_FMT-0\" # inset for receiver devices\n",
"path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
@@ -43,6 +42,7 @@
"limit_images = 0 # this parameter is used for limiting number of images to correct from a sequence file. ONLY FOR TESTING.\n",
"\n",
"# Conditions for retrieving calibration constants\n",
+ "set_parameters_manually = False # Boolean for setting parameter values manually instead of reading control values.\n",
"fix_temperature_top = 0. # fix temperature for top sensor in K, set to 0. to use value from slow data.\n",
"fix_temperature_bot = 0. # fix temperature for bottom senspr in K, set to 0. to use value from slow data.\n",
"gain = -1 # the detector's gain setting, It is later read from file and this value is overwritten\n",
@@ -111,7 +111,6 @@
"\n",
"%matplotlib inline\n",
"\n",
- "import XFELDetAna.xfelprofiler as xprof\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
"from cal_tools import pnccdlib\n",
@@ -124,10 +123,7 @@
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools import h5_copy_except\n",
"from iCalibrationDB import Conditions, ConstantMetaData, Constants\n",
- "from iCalibrationDB.detectors import DetectorTypes\n",
- "\n",
- "profiler = xprof.Profiler()\n",
- "profiler.disable()"
+ "from iCalibrationDB.detectors import DetectorTypes"
]
},
{
@@ -143,10 +139,6 @@
"print(f\"pnCCD size is: {pixels_x}x{pixels_y} pixels.\")\n",
"print(f'Calibration database interface selected: {cal_db_interface}')\n",
"\n",
- "proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
- "file_loc =f'Proposal: {proposal}, Run: {run}'\n",
- "print(file_loc)\n",
- "\n",
"# Paths to the data:\n",
"instrument_src = instrument_source_template.format(karabo_id, receiver_id)\n",
"print(f\"Instrument H5File source: {instrument_src}\\n\")\n",
@@ -184,11 +176,15 @@
"metadata": {},
"outputs": [],
"source": [
+ "run_dc = RunDirectory(Path(in_folder) / f\"r{run:04d}\")\n",
+ "ctrl_data = pnccdlib.PnccdCtrl(run_dc, karabo_id)\n",
+ "\n",
+ "proposal = run_dc.run_metadata()[\"proposalNumber\"]\n",
+ "print(f'Proposal: {proposal}, Run: {run}')\n",
+ "\n",
"# extract control data\n",
- "if karabo_da_control:\n",
+ "if not set_parameters_manually:\n",
" step_timer.start()\n",
- " run_dc = RunDirectory(Path(in_folder) / f\"r{run:04d}\")\n",
- " ctrl_data = pnccdlib.PnccdCtrl(run_dc, karabo_id)\n",
"\n",
" if bias_voltage == 0.:\n",
" bias_voltage = ctrl_data.get_bias_voltage()\n",
@@ -198,6 +194,7 @@
" fix_temperature_top = ctrl_data.get_fix_temperature_top()\n",
" if fix_temperature_bot == 0:\n",
" fix_temperature_bot = ctrl_data.get_fix_temperature_bot()\n",
+ "\n",
" step_timer.done_step(\"Reading control parameters.\")\n",
"\n",
"# Printing the Parameters Read from the Data File:\n",
@@ -315,8 +312,6 @@
"outputs": [],
"source": [
"# Sensor size and block size definitions (important for common mode and other corrections):\n",
- "run_parallel = False\n",
- "memoryCells = 1 # pnCCD has 1 memory cell\n",
"sensorSize = [pixels_x, pixels_y]\n",
"blockSize = [sensorSize[0]//2, sensorSize[1]//2] # sensor area will be analysed according to blockSize\n",
"xcal.defaultBlockSize = blockSize # for xcal.HistogramCalculators "
@@ -485,7 +480,7 @@
" cmCorrection = xcal.CommonModeCorrection([pixels_x, pixels_y],\n",
" blockSize,\n",
" commonModeAxis,\n",
- " parallel=run_parallel, dType=np.float32, stride=1,\n",
+ " parallel=False, dType=np.float32, stride=1,\n",
" noiseMap=constants[\"Noise\"].astype(np.float32), minFrac=0.25)\n",
"\n",
"if corr_bools.get('pattern_class'):\n",
@@ -497,10 +492,10 @@
" split_evt_secondary_threshold,\n",
" split_evt_mip_threshold,\n",
" tagFirstSingles=3, # track along y-axis, left to right (see \n",
- " nCells=memoryCells, # split_event.py file in pydetlib/lib/src/\n",
+ " nCells=1, # split_event.py file in pydetlib/lib/src/\n",
" allowElongated=False, # XFELDetAna/algorithms)\n",
" blockSize=[pixels_x, pixels_y//2],\n",
- " parallel=run_parallel)\n",
+ " parallel=False)\n",
"\n",
" # Right Hemisphere:\n",
" patternClassifierRH = xcal.PatternClassifier([pixels_x, pixels_y//2],\n",
@@ -509,10 +504,10 @@
" split_evt_secondary_threshold,\n",
" split_evt_mip_threshold,\n",
" tagFirstSingles=4, # track along y-axis, right to left\n",
- " nCells=memoryCells,\n",
+ " nCells=1,\n",
" allowElongated=False,\n",
" blockSize=[pixels_x, pixels_y//2],\n",
- " parallel=run_parallel)\n",
+ " parallel=False)\n",
"\n",
" patternClassifierLH._imagesPerChunk = 1\n",
" patternClassifierRH._imagesPerChunk = 1\n",
@@ -535,46 +530,46 @@
"histCalRaw = xcal.HistogramCalculator(sensorSize, \n",
" bins=bins, \n",
" range=bin_range,\n",
- " nCells=memoryCells, \n",
- " parallel=run_parallel,\n",
+ " nCells=1, \n",
+ " parallel=False,\n",
" blockSize=blockSize)\n",
"# Will contain offset corrected data:\n",
"histCalOffsetCor = xcal.HistogramCalculator(sensorSize, \n",
" bins=bins, \n",
" range=bin_range,\n",
- " nCells=memoryCells, \n",
- " parallel=run_parallel,\n",
+ " nCells=1, \n",
+ " parallel=False,\n",
" blockSize=blockSize)\n",
"if corr_bools.get('common_mode'):\n",
" # Will contain common mode corrected data:\n",
" histCalCommonModeCor = xcal.HistogramCalculator(sensorSize,\n",
" bins=bins,\n",
" range=bin_range,\n",
- " nCells=memoryCells, \n",
- " parallel=run_parallel,\n",
+ " nCells=1, \n",
+ " parallel=False,\n",
" blockSize=blockSize)\n",
"if corr_bools.get('pattern_class'):\n",
" # Will contain split events pattern data:\n",
" histCalPcorr = xcal.HistogramCalculator(sensorSize,\n",
" bins=bins,\n",
" range=bin_range,\n",
- " nCells=memoryCells, \n",
- " parallel=run_parallel,\n",
+ " nCells=1, \n",
+ " parallel=False,\n",
" blockSize=blockSize)\n",
" # Will contain singles events data:\n",
" histCalPcorrS = xcal.HistogramCalculator(sensorSize,\n",
" bins=bins,\n",
" range=bin_range,\n",
- " nCells=memoryCells, \n",
- " parallel=run_parallel,\n",
+ " nCells=1, \n",
+ " parallel=False,\n",
" blockSize=blockSize)\n",
"if corr_bools.get('relgain'):\n",
" # Will contain gain corrected data:\n",
" histCalGainCor = xcal.HistogramCalculator(sensorSize,\n",
" bins=bins,\n",
" range=bin_range,\n",
- " nCells=memoryCells, \n",
- " parallel=run_parallel,\n",
+ " nCells=1, \n",
+ " parallel=False,\n",
" blockSize=blockSize)"
]
},
--
GitLab