diff --git a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
index 9287f1fd196c629fa302cb434ff30e14690a79a1..2ebcc78c267bfcb45bac3b83db37af0446bba264 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
@@ -1,17 +1,12 @@
{
"cells": [
- {
- "cell_type": "raw",
- "metadata": {},
- "source": []
- },
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# pnCCD Dark Characterization\n",
"\n",
- "Author: DET Group, modified by Kiana Setoodehnia, Version 2.0\n",
+ "Author: DET Group, modified by Kiana Setoodehnia, Version: 2.0\n",
"\n",
"The following notebook provides dark image analysis of the pnCCD detector. Dark characterization evaluates offset and noise of the detector and gives information about bad pixels. \n",
"\n",
@@ -35,32 +30,40 @@
},
"outputs": [],
"source": [
+ "cluster_profile = \"noDB\" # ipcluster profile to use\n",
"in_folder = \"/gpfs/exfel/exp/SQS/201930/p900075/raw\" # input folder, required\n",
- "out_folder = '/gpfs/exfel/exp/SQS/201930/p900075/scratch' # output folder, required\n",
- "path_template = 'RAW-R{:04d}-PNCCD01-S{{:05d}}.h5' # the template to use to access data\n",
- "h5path = '/INSTRUMENT/SQS_NQS_PNCCD1MP/CAL/PNCCD_FMT-0:output/data/image/' # path to the data in the HDF5 file\n",
- "run = 364 # which run to read data from, required\n",
+ "out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/pnccd' # output folder, required\n",
"sequence = 0 # sequence file to use\n",
+ "run = 364 # which run to read data from, required\n",
+ "\n",
+ "karabo_da = 'PNCCD01' # data aggregators\n",
+ "karabo_id = \"SQS_NQS_PNCCD1MP\" # karabo prefix of PNCCD devices\n",
+ "receiver_id = \"PNCCD_FMT-0\" # inset for receiver devices\n",
+ "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
+ "h5path = '/INSTRUMENT/{}/CAL/{}:output/data/image/' # path in the HDF5 file the data is at\n",
+ "\n",
+ "# for database time derivation:\n",
+ "use_dir_creation_date = True # use dir creation date as data production reference date\n",
+ "cal_db_interface = \"tcp://max-exfl016:8021\" # calibration DB interface to use\n",
+ "cal_db_timeout = 300000 # timeout on caldb requests\n",
+ "db_output = False # if True, the notebook sends dark constants to the calibration database\n",
+ "local_output = True # if True, the notebook saves dark constants locally\n",
+ "\n",
"number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
"chunkSize = 100 # number of images to read per chunk\n",
- "fix_temperature = 233. # fix temperature in K, set to -1 to use value from slow data\n",
- "gain = 1 # the detector's gain setting, only 0 is currently implemented\n",
+ "fix_temperature = 233. # fix temperature in K, set to 0. to use value from slow data\n",
+ "gain = 1 # the detector's gain setting, only 1 and 64 is available.\n",
"bias_voltage = 300 # detector's bias voltage\n",
"integration_time = 70 # detector's integration time\n",
- "commonModeAxis = 'row' # axis along which common mode will be calculated (0: along rows, 1: along columns)\n",
+ "commonModeAxis = 0 # axis along which common mode will be calculated (0: along rows, 1: along columns)\n",
"commonModeBlockSize = [512, 512] # size of the detector in pixels for common mode calculations\n",
"sigmaNoise = 10. # pixels whose signal value exceeds sigmaNoise*noise will be considered as cosmics and are masked\n",
"bad_pixel_offset_sigma = 5. # any pixel whose offset is beyond 5 standard deviations, is a bad pixel\n",
"bad_pixel_noise_sigma = 5. # any pixel whose noise is beyond 5 standard deviations, is a bad pixel\n",
- "cluster_profile = \"noDB\" # ipcluster profile to use\n",
- "run_parallel = True # for parallel computation \n",
- "cpuCores = 40 # specifies the number of running cpu cores\n",
- "cal_db_interface = \"tcp://max-exfl016:8021\" # calibration DB interface to use\n",
"temp_limits = 5 # temperature limits in which calibration parameters are considered equal\n",
- "db_output = False # if True, the notebook sends dark constants to the calibration database\n",
- "local_output = True # if True, the notebook saves dark constants locally\n",
- "# for database time derivation:\n",
- "use_dir_creation_date = True # To be used to retrieve calibration constants later on"
+ "\n",
+ "run_parallel = True # for parallel computation\n",
+ "cpuCores = 40 # specifies the number of running cpu cores"
]
},
{
@@ -139,7 +142,7 @@
"source": [
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = 'Proposal: {}, Run: {}'.format(proposal, run)\n",
- "print(file_loc)"
+ "print(\"File Location:\", file_loc)"
]
},
{
@@ -156,14 +159,15 @@
"# Calibration Database Settings, and Some Initial Run Parameters & Paths:\n",
"\n",
"display(Markdown('### Initial Settings'))\n",
- "x = 1024 # rows of the FastCCD to analyze in FS mode \n",
- "y = 1024 # columns of the FastCCD to analyze in FS mode \n",
- "print(\"pnCCD size is: {}x{} pixels.\".format(x, y))\n",
- " \n",
+ "pixels_x = 1024 # rows of the FastCCD to analyze in FS mode \n",
+ "pixels_y = 1024 # columns of the FastCCD to analyze in FS mode \n",
+ "print(f\"pnCCD size is: {pixels_x}x{pixels_y} pixels.\")\n",
+ "\n",
"ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "fp_name = path_template.format(run)\n",
+ "fp_name = path_template.format(run, karabo_da)\n",
"fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
"filename = fp_path.format(sequence)\n",
+ "h5path = h5path.format(karabo_id, receiver_id)\n",
"\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
@@ -193,7 +197,7 @@
"# Reading Parameters such as Detector Bias, Gain, etc. from the Data:\n",
"\n",
"memoryCells = 1 # pnCCD has 1 memory cell\n",
- "sensorSize = [x, y]\n",
+ "sensorSize = [pixels_x, pixels_y]\n",
"blockSize = [sensorSize[0]//2, sensorSize[1]//2]# sensor area will be analysed according to blocksize\n",
"xcal.defaultBlockSize = blockSize\n",
"nImages = fastccdreaderh5.getDataSize(filename, h5path)[0] # specifies total number of images to proceed\n",
@@ -213,12 +217,15 @@
"print(\"Bias voltage is {} V.\".format(bias_voltage))\n",
"print(\"Detector gain is set to {}.\".format(gain))\n",
"print(\"Detector integration time is set to {} ms\".format(integration_time)) \n",
- " \n",
+ "\n",
"if fix_temperature != 0.:\n",
- " print(\"Using a fixed temperature of {} K\".format(fix_temperature))\n",
+ " print(f\"Using a fixed temperature of {fix_temperature} K\")\n",
" temperature_k = fix_temperature\n",
"else:\n",
" print(\"Temperature is not fixed.\")\n",
+ " #TODO: remove this line after properly saving the temperature in control data.\n",
+ " temperature_k = 233.\n",
+ " print(f\"Using a fixed temperature of {fix_temperature} K\")\n",
" \n",
"print(\"Number of dark images to analyze:\", nImages) "
]
@@ -237,8 +244,8 @@
"# Reading Files in Chunks:\n",
"\n",
"# Chunk reader returns an iterator to access the data in the file within the ranges:\n",
- "reader = ChunkReader(filename, fastccdreaderh5.readData, nImages, chunkSize, path = h5path, \n",
- " pixels_x = sensorSize[0], pixels_y = sensorSize[1],)"
+ "reader = ChunkReader(filename, fastccdreaderh5.readData, nImages, chunkSize, path=h5path, \n",
+ " pixels_x=pixels_x, pixels_y=pixels_y)"
]
},
{
@@ -387,12 +394,12 @@
"\n",
"#************** HEAT MAPS *******************#\n",
"fig = xana.heatmapPlot(offsetMap[:,:,0], x_label='Column Number', y_label='Row Number', aspect=1,\n",
- " x_range=(0, y), y_range=(0, x), vmin=6000, vmax=14500, lut_label='Offset (ADU)', \n",
+ " x_range=(0, pixels_y), y_range=(0, pixels_x), vmin=6000, vmax=14500, lut_label='Offset (ADU)', \n",
" panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', title = 'Offset Map')\n",
"\n",
"fig = xana.heatmapPlot(noiseMap[:,:,0], x_label='Column Number', y_label='Row Number', aspect=1,\n",
- " lut_label='Noise (ADU)', x_range=(0, y),\n",
- " y_range=(0, x), vmax=2*np.mean(noiseMap), panel_x_label='Row Stat (ADU)', \n",
+ " lut_label='Noise (ADU)', x_range=(0, pixels_y),\n",
+ " y_range=(0, pixels_x), vmax=2*np.mean(noiseMap), panel_x_label='Row Stat (ADU)', \n",
" panel_y_label='Column Stat (ADU)', title = 'Uncorrected NoiseMap')"
]
},
@@ -425,8 +432,8 @@
"fig = xana.heatmapPlot(np.log2(bad_pixels[:, :, 0]),\n",
" x_label='Columns', y_label='Rows',\n",
" lut_label='Bad Pixel Value (ADU)',\n",
- " x_range=(0, y),\n",
- " y_range=(0, x), vmin=0, vmax=32,\n",
+ " x_range=(0, pixels_y),\n",
+ " y_range=(0, pixels_x), vmin=0, vmax=32,\n",
" panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', \n",
" title = 'Initial Bad Pixels Map')"
]
@@ -446,7 +453,7 @@
"# Common Mode Correction:\n",
"# In this method, the median of all pixels that are read out at the same time along a row is subtracted from the\n",
"# signal in each pixel:\n",
- "cmCorrection = xcal.CommonModeCorrection([x, y],\n",
+ "cmCorrection = xcal.CommonModeCorrection(sensorSize,\n",
" commonModeBlockSize,\n",
" commonModeAxis, parallel=run_parallel, dType=np.float32, stride=1,\n",
" noiseMap=noiseMap.astype(np.float32), minFrac=0)\n",
@@ -562,15 +569,12 @@
"source": [
"do = [{'x': co,\n",
" 'y': ho,\n",
- " #'y_err': np.sqrt(ho[:]),\n",
- " #'drawstyle': 'steps-mid',\n",
" 'drawstyle': 'steps-post',\n",
" 'color': 'cornflowerblue',\n",
" 'label': 'Offset Corrected Signal'\n",
" },\n",
" {'x': cCM,\n",
" 'y': hCM,\n",
- " #'y_err': np.sqrt(hCM[:]),\n",
" 'drawstyle': 'steps-post',\n",
" 'color': 'red',\n",
" 'label': 'Common Mode Corrected Signal'\n",
@@ -614,17 +618,14 @@
"\n",
"dn = [{'x': cn[:-1],\n",
" 'y': hn,\n",
- " # 'y_err': np.sqrt(hn[:]),\n",
- " 'drawstyle': 'steps-post', # 'bars' or 'steps-mid',\n",
- " 'color': 'blue', # 'cornflowerblue',\n",
+ " 'drawstyle': 'steps-post',\n",
+ " 'color': 'blue',\n",
" 'label': 'Uncorrected'\n",
" },\n",
" {'x': cn_CM[:-1],\n",
" 'y': hn_CM,\n",
- " # 'y_err': np.sqrt(hn_CM[:]),\n",
" 'drawstyle': 'steps-post', # 'bars',\n",
" 'color': 'crimson', # 'red',#'cornflowerblue',\n",
- " # 'ecolor': 'crimson',\n",
" 'label': 'Common Mode Corrected'\n",
" }]\n",
"fig = xana.simplePlot(dn, figsize='1col', aspect=1, x_label='Noise (ADU)', y_label=\"Counts\",\n",
@@ -635,8 +636,8 @@
"fig = xana.heatmapPlot(noiseMapCM[:, :, 0],\n",
" x_label='Columns', y_label='Rows',\n",
" lut_label='Common Mode Corrected Noise (ADU)',\n",
- " x_range=(0, y),\n",
- " y_range=(0, x), vmax=2*np.mean(noiseMapCM), panel_x_label='Row Stat (ADU)', \n",
+ " x_range=(0, pixels_y),\n",
+ " y_range=(0, pixels_x), vmax=2*np.mean(noiseMapCM), panel_x_label='Row Stat (ADU)', \n",
" panel_y_label='Column Stat (ADU)', title='Common Mode Corrected Noise Map')"
]
},
@@ -681,8 +682,8 @@
"fig = xana.heatmapPlot(np.log2(bad_pixels[:, :, 0]),\n",
" x_label='Columns', y_label='Rows',\n",
" lut_label='Bad Pixel Value (ADU)',\n",
- " x_range=(0, y),\n",
- " y_range=(0, x), vmin=0, vmax=32,\n",
+ " x_range=(0, pixels_y),\n",
+ " y_range=(0, pixels_x), vmin=0, vmax=32,\n",
" panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', \n",
" title = 'Second Bad Pixels Map')"
]
@@ -703,7 +704,6 @@
"outputs": [],
"source": [
"hole_mask = np.zeros(bad_pixels.shape, np.uint32) \n",
- "#hole_mask[485:539,481:543,:] = BadPixels.NON_SENSITIVE.value # This is from Markus estimate\n",
"hole_mask[483:539,477:543,:] = BadPixels.NON_SENSITIVE.value\n",
"\n",
"# Assigning this masked area as bad pixels:\n",
@@ -712,8 +712,8 @@
"fig = xana.heatmapPlot(np.log2(bad_pixels[:, :, 0]),\n",
" x_label='Columns', y_label='Rows',\n",
" lut_label='Bad Pixel Value (ADU)',\n",
- " x_range=(0, y),\n",
- " y_range=(0, x), vmin=0, vmax=32, panel_x_label='Row Stat (ADU)', \n",
+ " x_range=(0, pixels_y),\n",
+ " y_range=(0, pixels_x), vmin=0, vmax=32, panel_x_label='Row Stat (ADU)', \n",
" panel_y_label='Column Stat (ADU)', title = 'Second Bad Pixels Map with the Hole in Center')"
]
},
@@ -812,35 +812,27 @@
"source": [
"do_Final = [{'x': co_second_trial,\n",
" 'y': ho_second_trial,\n",
- " #'y_err': np.sqrt(ho_second_trial[:]),\n",
" 'drawstyle': 'steps-post',\n",
- " 'color': 'blue',#'cornflowerblue',\n",
- " #'errorstyle': 'bars',\n",
+ " 'color': 'blue',\n",
" 'label': 'Offset Correction, Bad Pixels Included - 2nd Trial'\n",
" },\n",
" {'x': cCM_second_trial,\n",
" 'y': hCM_second_trial,\n",
- " #'y_err': np.sqrt(hCM_second_trial[:]),\n",
" 'drawstyle': 'steps-post',\n",
" 'color': 'red',\n",
- " #'errorstyle': 'bars',\n",
" 'ecolor': 'crimson',\n",
" 'label': 'Common Mode Correction, Bad Pixels Included - 2nd Trial' \n",
" },\n",
" {'x': co2,\n",
" 'y': ho2,\n",
- " #'y_err': np.sqrt(ho2[:]),\n",
" 'drawstyle': 'steps-post',\n",
- " 'color': 'black', #'cornflowerblue',\n",
- " #'errorstyle': 'bars',\n",
+ " 'color': 'black',\n",
" 'label': 'Offset Correction, Bad Pixels Excluded - 3rd Trial'\n",
" },\n",
" {'x': cCM2,\n",
" 'y': hCM2,\n",
- " #'y_err': np.sqrt(hCM2[:]),\n",
" 'drawstyle': 'steps-post',\n",
- " 'color': 'orange', #'cornflowerblue',\n",
- " #'errorstyle': 'bars',\n",
+ " 'color': 'orange',\n",
" 'label': 'Common Mode Correction, Bad Pixels Excluded - 3rd Trial'\n",
" }]\n",
"\n",
@@ -880,24 +872,20 @@
"\n",
"dn2 = [{'x': cn[:-1],\n",
" 'y': hn,\n",
- " #'y_err': np.sqrt(hn[:]),\n",
- " 'drawstyle': 'steps-post',#'bars',\n",
- " 'color': 'blue', #'cornflowerblue',\n",
+ " 'drawstyle': 'steps-post',\n",
+ " 'color': 'blue',\n",
" 'label': 'Uncorrected'\n",
" },\n",
" {'x': cn_CM[:-1],\n",
" 'y': hn_CM,\n",
- " #'y_err': np.sqrt(hn_CM[:]),\n",
" 'drawstyle': 'steps-post',\n",
" 'color': 'red',\n",
- " #'ecolor': 'crimson',\n",
" 'label': 'Common Mode Corrected prior to Bad Pixels Exclusion'\n",
" },\n",
" {'x': cn_CM2[:-1],\n",
" 'y': hn_CM2,\n",
- " #'y_err': np.sqrt(hn_CM2[:]),\n",
" 'drawstyle': 'steps-post',\n",
- " 'color': 'black', #'cornflowerblue',\n",
+ " 'color': 'black',\n",
" 'label': 'Common Mode Corrected after Bad Pixels Exclusion'\n",
" }]\n",
"\n",
@@ -906,7 +894,7 @@
" title = 'Final Noise Comparison')\n",
"\n",
"fig = xana.heatmapPlot(noiseMapCM_2nd[:,:,0], aspect=1, x_label='Column Number', y_label='Row Number',\n",
- " lut_label='Noise (ADU)', x_range=(0, y), y_range=(0, x),\n",
+ " lut_label='Noise (ADU)', x_range=(0, pixels_y), y_range=(0, pixels_x),\n",
" title = 'Final Common Mode Corrected Noise\\n (Bad Pixels Excluded)', \n",
" panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)')"
]
@@ -943,8 +931,8 @@
"fig = xana.heatmapPlot(np.log2(bad_pixels[:, :, 0]),\n",
" x_label='Columns', y_label='Rows',\n",
" lut_label='Bad Pixel Value (ADU)',\n",
- " x_range=(0, y),\n",
- " y_range=(0, x), vmin=0, vmax=32, panel_x_label='Row Stat (ADU)', \n",
+ " x_range=(0, pixels_y),\n",
+ " y_range=(0, pixels_x), vmin=0, vmax=32, panel_x_label='Row Stat (ADU)', \n",
" panel_y_label='Column Stat (ADU)', title = 'Final Bad Pixels Map')"
]
},
@@ -956,7 +944,7 @@
"source": [
"display(Markdown('### Statistics on the Bad Pixels'))\n",
"num_bad_pixels = np.count_nonzero(bad_pixels)\n",
- "num_all_pixels = x*y\n",
+ "num_all_pixels = pixels_x*pixels_y\n",
"percentage_bad_pixels = num_bad_pixels*100/num_all_pixels\n",
"print(\"Number of bad pixels: {:0.0f}, i.e. {:0.2f}% of all pixels\".format(num_bad_pixels, percentage_bad_pixels))"
]
@@ -1000,8 +988,8 @@
" integration_time=integration_time,\n",
" gain_setting=gain,\n",
" temperature=temperature_k,\n",
- " pixels_x=1024,\n",
- " pixels_y=1024)\n",
+ " pixels_x=pixels_x,\n",
+ " pixels_y=pixels_y)\n",
"\n",
" device = Detectors.PnCCD1\n",
" metadata.detector_condition = condition\n",
@@ -1018,22 +1006,19 @@
" if db_output:\n",
" try:\n",
" metadata.send(cal_db_interface)\n",
- " print(\"Inject {} constants from {}\".format(const_name,\n",
- " metadata.calibration_constant_version.begin_at))\n",
+ " print(f\"Inject {const_name} constants from {metadata.calibration_constant_version.begin_at}\")\n",
" except Exception as e:\n",
" print(e)\n",
"\n",
" if local_output:\n",
" save_const_to_h5(metadata, out_folder)\n",
- " print(\"Calibration constant {} is stored to {}.\".format(const, out_folder))"
+ " print(f\"Calibration constant {const.name} is stored to {out_folder}.\\n\")\n",
+ "\n",
+ "print(\"Generated constants with conditions:\\n\")\n",
+ "print(f\"• bias_voltage: {bias_voltage}\\n• integration_time: {integration_time}\\n\"\n",
+ " f\"• gain_setting: {gain}\\n• temperature: {temperature_k}\\n\"\n",
+ " f\"• creation_time: {creation_time}\\n\")"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {