diff --git a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..97e862ab69d984e62db60a28b0808f0e2f1b1023
--- /dev/null
+++ b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
@@ -0,0 +1,1003 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# FastCCD Dark Characterization\n",
+ "\n",
+ "Author: I. KlaÄková, S. Hauf, K. Setoodehnia and M. Cascella\n",
+ "\n",
+ "The following notebook provides dark image analysis of the FastCCD detector.\n",
+ "\n",
+ "Dark characterization evaluates offset and noise of the FastCCD detector, corrects the noise for Common Mode (CM), and defines bad pixels relative to offset and CM corrected noise. Bad pixels are then excluded and CM corrected noise is recalculated excluding the bad pixels. Resulting offset and CM corrected noise maps, as well as the bad pixel map are sent to the calibration database."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2018-12-06T10:54:38.999974Z",
+ "start_time": "2018-12-06T10:54:38.983406Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Initial Parameters:\n",
+ "\n",
+ "in_folder = \"/gpfs/exfel/exp/SCS/201930/p900074/raw\" # input folder, required\n",
+ "out_folder = '/gpfs/exfel/data/scratch/setoodeh/DarkRuns' # output folder, required\n",
+ "path_template = 'RAW-R{:04d}-DA05-S{{:05d}}.h5' # the template to use to access data\n",
+ "run = 351 # which run to read data from, required\n",
+ "number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
+ "cluster_profile = \"noDB\" # ipcluster profile to use\n",
+ "# The two operation modes for FastCCD have fixed names which cannot be changed:\n",
+ "operation_mode = \"FF\" # FS stands for frame-store and FF for full-frame opeartion. \n",
+ "h5path = '/INSTRUMENT/SCS_CDIDET_FCCD2M/DAQ/FCCD:daqOutput/data/image/pixels' # path to the data in the HDF5 file \n",
+ "h5path_t = '/CONTROL/SCS_CDIDET_FCCD2M/CTRL/LSLAN/inputA/crdg/value' # path to find temperature\n",
+ "h5path_cntrl = '/RUN/SCS_CDIDET_FCCD2M/DET/FCCD' # path to find control data\n",
+ "cal_db_interface = \"tcp://max-exfl016:8020\" # the calibration database interface to use\n",
+ "cal_db_timeout = 300000 # timeout on calibration database requests\n",
+ "temp_limits = 5 # to find calibration constants later on, the sensor temperature is allowed to vary by 5 units\n",
+ "sequence = 0 # sequallence file to use\n",
+ "use_dir_creation_date = True # To be used to retrieve calibration constants later on (for database time derivation)\n",
+ "bad_pixel_offset_sigma = 5. # Any pixel whose offset is beyond 5 standard deviations, is a bad pixel\n",
+ "bad_pixel_noise_sigma = 5. # Any pixel whose noise is beyond 5 standard deviations, is a bad pixel\n",
+ "sigmaNoise = 5. # Any pixel whose signal exceeds 'sigmaNoise'*noiseCM (common mode corrected noise) will be masked\n",
+ "fix_temperature = False # This is required when the temperature is fixed in calibration database\n",
+ "temperature_k = 233 # This is only used in the case of a fixed temperature for the calibration database\n",
+ "chunkSize = 100 # Number of images to read per chunk\n",
+ "cpuCores = 40 # Specifies the number of running cpu cores\n",
+ "commonModeAxis = 1 # Axis along which common mode will be calculated (0: along rows, 1: along columns)\n",
+ "ADU_to_electron_upper = 6.1 # According to Table 6.1 of Ivana KlaÄková's master's thesis, for upper hemisphere: conversion\n",
+ " # gain is 1 ADU = 6.1e-\n",
+ "ADU_to_electron_lower = 6.2 # and for lower hemisphere: conversion gain is 1 ADU = 6.2e-\n",
+ "run_parallel = True # For parallel computation \n",
+ "db_output = True # Output constants to the calibration database"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2018-12-06T10:54:39.467334Z",
+ "start_time": "2018-12-06T10:54:39.427784Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Required Packages:\n",
+ "\n",
+ "import copy\n",
+ "import datetime\n",
+ "import os\n",
+ "import time\n",
+ "import warnings\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "import h5py\n",
+ "from IPython.display import display, Markdown\n",
+ "import matplotlib.pyplot as plt\n",
+ "%matplotlib inline\n",
+ "import numpy as np\n",
+ "from prettytable import PrettyTable\n",
+ "\n",
+ "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
+ "from iCalibrationDB.detectors import DetectorTypes\n",
+ "from cal_tools.tools import get_dir_creation_date\n",
+ "from cal_tools.enums import BadPixels\n",
+ "from XFELDetAna import xfelpyanatools as xana\n",
+ "from XFELDetAna import xfelpycaltools as xcal\n",
+ "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
+ "from XFELDetAna.util import env\n",
+ "env.iprofile = cluster_profile\n",
+ "import XFELDetAna.xfelprofiler as xprof\n",
+ "profiler = xprof.Profiler()\n",
+ "profiler.disable()\n",
+ "from XFELDetAna.xfelreaders import ChunkReader"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2018-12-06T10:54:39.467334Z",
+ "start_time": "2018-12-06T10:54:39.427784Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Output Folder Creation:\n",
+ "if not os.path.exists(out_folder):\n",
+ " os.makedirs(out_folder)\n",
+ "\n",
+ "# Number of Images:\n",
+ "def nImagesOrLimit(nImages, limit):\n",
+ " if limit == 0:\n",
+ " return nImages\n",
+ " else:\n",
+ " return min(nImages, limit)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2018-12-06T10:54:40.058101Z",
+ "start_time": "2018-12-06T10:54:40.042615Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Detector Operation Mode, Calibration Database Settings, and Some Initial Run Parameters & Paths:\n",
+ "\n",
+ "display(Markdown('### Initial Settings'))\n",
+ "if operation_mode == \"FS\":\n",
+ " x = 960 # rows of the FastCCD to analyze in FS mode \n",
+ " y = 960 # columns of the FastCCD to analyze in FS mode \n",
+ " print('\\nYou are analyzing data in FS mode.')\n",
+ "else:\n",
+ " x = 1934 # rows of the FastCCD to analyze in FF mode \n",
+ " y = 960 # columns of the FastCCD to analyze in FF mode\n",
+ " print('\\nYou are analyzing data in FF mode.')\n",
+ " \n",
+ "ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
+ "fp_name = path_template.format(run)\n",
+ "fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
+ "filename = fp_path.format(sequence)\n",
+ "\n",
+ "creation_time = None\n",
+ "if use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run)\n",
+ " \n",
+ "print('Calibration database Interface: {}'.format(cal_db_interface))\n",
+ "print(\"Sending constants to the calibration database: {}\".format(db_output))\n",
+ "print(\"HDF5 path to data: {}\".format(h5path))\n",
+ "print(\"Run number: {}\".format(run))\n",
+ "print(\"Reading data from: {}\".format(filename))\n",
+ "if creation_time:\n",
+ " print(\"Using {} as creation time\".format(creation_time.isoformat()))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2018-12-06T10:54:40.555804Z",
+ "start_time": "2018-12-06T10:54:40.452978Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Reading Parameters such as Detector Bias, Gain, etc. from the Data:\n",
+ "\n",
+ "memoryCells = 1 # FastCCD has 1 memory cell\n",
+ "sensorSize = [x, y]\n",
+ "blockSize = [sensorSize[0]//2, sensorSize[1]] # Sensor area will be analysed according to blocksize\n",
+ "xcal.defaultBlockSize = blockSize\n",
+ "nImages = fastccdreaderh5.getDataSize(filename, h5path)[0] # Specifies total number of images to proceed\n",
+ "nImages = nImagesOrLimit(nImages, number_dark_frames)\n",
+ "profile = False\n",
+ "gain_setting = None\n",
+ "\n",
+ "with h5py.File(filename, 'r') as f:\n",
+ " bias_voltage = int(f['{}/biasclock/bias/value'.format(h5path_cntrl)][0])\n",
+ " det_gain = int(f['{}/exposure/gain/value'.format(h5path_cntrl)][0])\n",
+ " integration_time = int(f['{}/acquisitionTime/value'.format(h5path_cntrl)][0])\n",
+ " temperature = np.mean(f[h5path_t])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2018-12-06T10:54:40.555804Z",
+ "start_time": "2018-12-06T10:54:40.452978Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Printing the Parameters Read from the Data File:\n",
+ "\n",
+ "display(Markdown('### Evaluated Parameters'))\n",
+ "print(\"Number of dark images to analyze:\",nImages) \n",
+ "\n",
+ "if det_gain == 8:\n",
+ " gain_setting = \"high\"\n",
+ "elif det_gain == 2:\n",
+ " gain_setting = \"medium\"\n",
+ "elif det_gain == 1:\n",
+ " gain_setting = \"low\"\n",
+ "else:\n",
+ " gain_setting = \"auto\"\n",
+ "\n",
+ "print(\"Bias voltage is {} V\".format(bias_voltage))\n",
+ "print(\"Detector gain is set to x{}\".format(det_gain), \"({} gain)\".format(gain_setting))\n",
+ "print(\"Detector integration time is set to {}\".format(integration_time), 'ms') \n",
+ " \n",
+ "if fix_temperature:\n",
+ " print(\"Using a fixed temperature of {}\".format(temperature_k), \"K\")\n",
+ " print(\"Mean temperature was {:0.2f} °C / {:0.2f} K\".format(temperature, temperature + 273.15))\n",
+ "else:\n",
+ " temperature_k = temperature + 273.15\n",
+ " print(\"Temperature is not fixed.\")\n",
+ " print(\"Mean temperature was {:0.2f} °C / {:0.2f} K\".format(temperature, temperature_k))\n",
+ "\n",
+ "print(\"Output: {}\".format(out_folder))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2018-12-06T10:54:41.899511Z",
+ "start_time": "2018-12-06T10:54:41.864816Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Reading Files in Chunks:\n",
+ "\n",
+ "# Chunk reader returns an iterator to access the data in the file within the ranges:\n",
+ "\n",
+ "reader = ChunkReader(filename, fastccdreaderh5.readData, nImages, chunkSize, path = h5path, pixels_x = sensorSize[0],\n",
+ " pixels_y = sensorSize[1],)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2018-12-06T10:54:41.899511Z",
+ "start_time": "2018-12-06T10:54:41.864816Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Calculator:\n",
+ "\n",
+ "# noiseCal is a noise map calculator, which internally also produces a per-pixel mean map, i.e. an offset map: \n",
+ " \n",
+ "noiseCal = xcal.NoiseCalculator(sensorSize, memoryCells, cores=cpuCores, blockSize=blockSize, runParallel=run_parallel)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### First Iteration\n",
+ "\n",
+ "Characterization of dark images with purpose to create dark maps (offset, noise and bad pixel maps) is an iterative process. Firstly, initial offset and noise maps are produced from raw dark data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2018-12-06T10:55:21.238009Z",
+ "start_time": "2018-12-06T10:54:54.586435Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "for data in reader.readChunks():\n",
+ " data = np.bitwise_and(data.astype(np.uint16), 0b0011111111111111).astype(np.float32)\n",
+ " dx = np.count_nonzero(data, axis=(0, 1))\n",
+ " data = data[:,:,dx != 0]\n",
+ " noiseCal.fill(data) # Filling calculators with data\n",
+ " \n",
+ "offsetMap = noiseCal.getOffset() # Producing offset map\n",
+ "noiseMap = noiseCal.get() # Producing noise map\n",
+ "noiseCal.reset() # Resetting noise calculator\n",
+ "print(\"Initial maps are created.\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Offset and Noise Maps prior to Common Mode Correction\n",
+ "\n",
+ "In the following, the histogram of the FastCCD offset, FastCCD offset map, as well as the initial uncorrected noise map are plotted:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#************** OFFSET MAP HISTOGRAM ***********#\n",
+ "ho,co = np.histogram(offsetMap.flatten(), bins=700) # ho = offset histogram; co = offset bin centers\n",
+ "do = {'x': co[:-1],\n",
+ " 'y': ho,\n",
+ " 'y_err': np.sqrt(ho[:]),\n",
+ " 'drawstyle': 'bars',\n",
+ " 'color': 'cornflowerblue',\n",
+ " 'label': 'Raw Signal (ADU)'\n",
+ " }\n",
+ "fig = xana.simplePlot(do, figsize='1col', aspect=1, x_label = 'Raw Signal (ADU)', y_label=\"Counts\", \n",
+ " x_range = (3400,4000), title = 'Offset Histogram')\n",
+ "#fig.savefig('Offset_Hist.svg', format='svg', dpi=1200, bbox_inches='tight') \n",
+ "\n",
+ "t0 = PrettyTable()\n",
+ "t0.title = \"Raw Signal\"\n",
+ "t0.field_names = [\"Mean\",\"Median\", \"Standard Deviation\"]\n",
+ "t0.add_row([\"{:0.3f} (ADU)\".format(np.mean(data)), \"{:0.3f} (ADU)\".format(np.median(data)), \"{:0.3f} (ADU)\".format(np.std(data))])\n",
+ "print(t0,'\\n')\n",
+ "\n",
+ "#************** OffsetMAP *******************#\n",
+ "fig = xana.heatmapPlot(offsetMap[:,:,0], x_label='Column Number', y_label='Row Number', aspect=1,\n",
+ " x_range=(0,y), y_range=(0,x), vmin=3000, vmax=4300, lut_label='Offset (ADU)', \n",
+ " panel_x_label='Columns Stat (ADU)', panel_y_label='Rows Stat (ADU)', \n",
+ " panel_top_low_lim = 3000, panel_top_high_lim = 4500, panel_side_low_lim = 3000, \n",
+ " panel_side_high_lim = 5000, title = 'OffsetMap')\n",
+ "#fig.savefig('RawOffsetMap.pdf', format='pdf', dpi=400, bbox_inches='tight')\n",
+ "\n",
+ "#************** Raw NoiseMAP *******************#\n",
+ "fig = xana.heatmapPlot(noiseMap[:,:,0], x_label='Column Number', y_label='Row Number', aspect=1,\n",
+ " lut_label='Uncorrected Noise (ADU)', x_range=(0,y),\n",
+ " y_range=(0,x), vmax=2*np.mean(noiseMap), panel_x_label='Columns Stat (ADU)', \n",
+ " panel_y_label='Rows Stat (ADU)', panel_top_low_lim = 0, panel_top_high_lim = 20, \n",
+ " panel_side_low_lim = 0, panel_side_high_lim = 50, title = 'Uncorrected NoiseMap')\n",
+ "#fig.savefig('RawNoiseMap.pdf', format='pdf', dpi=400, bbox_inches='tight')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Offset Correction:\n",
+ "\n",
+ "offsetCorrection = xcal.OffsetCorrection(sensorSize, offsetMap, nCells = memoryCells, cores=cpuCores, gains=None,\n",
+ " runParallel=run_parallel, blockSize=blockSize)\n",
+ "\n",
+ "offsetCorrection.debug()\n",
+ "\n",
+ "# Common Mode Correction:\n",
+ "# This is the new method subtracting the median of all pixels that are read out at the same time along a row:\n",
+ "cmCorrection = xcal.CommonModeCorrection([data.shape[0], data.shape[1]], [data.shape[0]//2, data.shape[1]], \n",
+ " commonModeAxis, parallel=False, dType=np.float32, stride=10,\n",
+ " noiseMap=noiseMap.astype(np.float32), minFrac=0)\n",
+ "\n",
+ "cmCorrection.debug()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Histogram Calculators:\n",
+ "\n",
+ "# For offset corrected data:\n",
+ "histCalCorrected = xcal.HistogramCalculator(sensorSize, bins=600, range=[-200, 200], memoryCells=memoryCells, \n",
+ " cores=cpuCores, gains=None, blockSize=blockSize)\n",
+ "# For common mode corrected data:\n",
+ "histCalCMCorrected = xcal.HistogramCalculator(sensorSize, bins=600, range=[-200, 200], memoryCells=memoryCells, \n",
+ " cores=cpuCores, gains=None, blockSize=blockSize)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Second Iteration\n",
+ "\n",
+ "During the second iteration, the data are offset corrected and then common mode corrected to produced a common mode corrected noise map. The common mode correction is calculated by subtracting out the median of all pixels that are read out at the same time along a row."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for data in reader.readChunks():\n",
+ " \n",
+ " data = data.astype(np.float32)\n",
+ " dx = np.count_nonzero(data, axis=(0, 1))\n",
+ " data = data[:,:,dx != 0] \n",
+ " data = offsetCorrection.correct(data) # Offset correction\n",
+ " offset_corr_data = copy.copy(data) # I am copying this so that I can have access to it in the table below \n",
+ " histCalCorrected.fill(data)\n",
+ " cellTable=np.zeros(data.shape[2], np.int32) # Common mode correction\n",
+ " data = cmCorrection.correct(data.astype(np.float32), cellTable=cellTable) # Common mode correction\n",
+ " histCalCMCorrected.fill(data)\n",
+ " noiseCal.fill(data) # Filling noise calculator with common mode (CM) corrected data\n",
+ " \n",
+ "noiseMapCM = noiseCal.get() # Produces CM corrected noise map\n",
+ "ho, eo, co , so = histCalCorrected.get()\n",
+ "hCM, eCM, cCM ,sCM = histCalCMCorrected.get()\n",
+ "print(\"Offset and common mode corrections are applied.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# I am copying these so that I can replot them later after the calculators are reset:\n",
+ "\n",
+ "ho_second_trial = copy.copy(ho)\n",
+ "co_second_trial = copy.copy(co)\n",
+ "hCM_second_trial = copy.copy(hCM)\n",
+ "cCM_second_trial = copy.copy(cCM)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Signal after Offset and Common Mode Corrections\n",
+ "\n",
+ "Here, the offset corrected signal is compared to the common-mode corrected signal (in the form of binned histograms): "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "do = [{'x': co,\n",
+ " 'y': ho,\n",
+ " 'y_err': np.sqrt(ho[:]),\n",
+ " 'drawstyle': 'steps-mid',\n",
+ " 'color': 'cornflowerblue',\n",
+ " 'label': 'Offset Corrected Signal'\n",
+ " },\n",
+ " {'x': cCM,\n",
+ " 'y': hCM,\n",
+ " 'y_err': np.sqrt(hCM[:]),\n",
+ " 'drawstyle': 'steps-mid',\n",
+ " 'color': 'red',\n",
+ " 'label': 'Common Mode Corrected Signal'\n",
+ " }]\n",
+ " \n",
+ "fig = xana.simplePlot(do, figsize='2col', aspect=1, x_label = 'Corrected Signal (ADU)', y_label=\"Counts\", \n",
+ " x_range = (-20,20), legend='top-right-frame-1col', title = 'Corrected Signal - 2nd Iteration')\n",
+ "#fig.savefig('Corrected_Signal_Hist_1.svg', format='svg', dpi=1200, bbox_inches='tight') \n",
+ "\n",
+ "t0 = PrettyTable()\n",
+ "t0.title = \"Comparison of the First Round of Corrections - Bad Pixels Included\"\n",
+ "t0.field_names = [\"After Offset Correction\",\"After Common Mode Correction\"]\n",
+ "t0.add_row([\"Mean: {:0.3f} (ADU)\".format(np.mean(offset_corr_data)), \"Mean: {:0.3f} (ADU)\".format(np.mean(data))])\n",
+ "t0.add_row([\"Median: {:0.3f} (ADU)\".format(np.median(offset_corr_data)), \"Median: {:0.3f} (ADU)\".format(np.median(data))])\n",
+ "t0.add_row([\"Standard Deviation: {:0.3f} (ADU)\".format(np.std(offset_corr_data)), \"Standard Deviation: {:0.3f} (ADU)\".format(np.std(data))])\n",
+ "print(t0,'\\n')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Noise Map after Common Mode Correction\n",
+ "\n",
+ "In the following, the effect of common mode correction on the noise is shown. Finally common mode corrected noise map (noiseMapCM) is displayed and compared to the initial uncorrected noise map:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "#*****NOISE MAP HISTOGRAM FROM THE COMMON MODE CORRECTED DATA*******#\n",
+ "hn,cn = np.histogram(noiseMap.flatten(), bins=200, range=(2,40)) # hn: histogram of noise, cn: bin centers for noise\n",
+ "hn_CM,cn_CM = np.histogram(noiseMapCM.flatten(), bins=200, range=(2,40))\n",
+ "\n",
+ "dn = [{'x': cn[:-1],\n",
+ " 'y': hn,\n",
+ " #'y_err': np.sqrt(hn[:]),\n",
+ " 'drawstyle': 'steps-mid',#'bars',\n",
+ " 'color': 'blue',#'cornflowerblue',\n",
+ " 'label': 'Uncorrected Noise'\n",
+ " },\n",
+ " {'x': cn_CM[:-1],\n",
+ " 'y': hn_CM,\n",
+ " #'y_err': np.sqrt(hn_CM[:]),\n",
+ " 'drawstyle': 'steps-mid',#'bars',\n",
+ " 'color': 'crimson',#'red',#'cornflowerblue',\n",
+ " #'ecolor': 'crimson',\n",
+ " 'label': 'Common Mode Corrected Noise'\n",
+ " }]\n",
+ "fig = xana.simplePlot(dn, figsize='2col', aspect=1, x_label = 'Noise (ADU)', y_label=\"Counts\", \n",
+ " x_range=(0,40), y_range=(0,1e6), y_log=True, legend='top-center-frame-1col',\n",
+ " title = 'Noise Comparison')\n",
+ "\n",
+ "#fig.savefig('Noise_CM_1_Hist.svg', format='svg', dpi=1200, bbox_inches='tight') \n",
+ "\n",
+ "fig = xana.heatmapPlot(noiseMapCM[:,:,0], aspect=1, x_label='Column Number', y_label='Row Number',\n",
+ " lut_label='Common Mode Corrected Noise (ADU)', x_range=(0,y), y_range=(0,x), \n",
+ " vmax=2*np.mean(noiseMapCM), panel_top_low_lim = 0, panel_top_high_lim = 20, panel_side_low_lim = 0,\n",
+ " panel_side_high_lim = 50, title = 'Common Mode Corrected Noise', \n",
+ " panel_x_label='Columns Stat (ADU)', panel_y_label='Rows Stat (ADU)')\n",
+ "\n",
+ "#fig.savefig('NoiseMapCM.pdf', format='pdf', dpi=400, bbox_inches='tight')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Resetting the calculators so we can do a third iteration later:\n",
+ "\n",
+ "noiseCal.reset()\n",
+ "histCalCorrected.reset()\n",
+ "histCalCMCorrected.reset()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Initial BadPixelMap\n",
+ "This is generated based on the offset and CM corrected noise maps:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "bad_pixels = np.zeros(offsetMap.shape, np.uint32)\n",
+ "mnoffset = np.nanmedian(offsetMap)\n",
+ "stdoffset = np.nanstd(offsetMap)\n",
+ "bad_pixels[(offsetMap < mnoffset-bad_pixel_offset_sigma*stdoffset) | \n",
+ " (offsetMap > mnoffset+bad_pixel_offset_sigma*stdoffset)] = BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
+ "\n",
+ "mnnoise = np.nanmedian(noiseMapCM)\n",
+ "stdnoise = np.nanstd(noiseMapCM)\n",
+ "bad_pixels[(noiseMapCM < mnnoise-bad_pixel_noise_sigma*stdnoise) | \n",
+ " (noiseMapCM > mnnoise+bad_pixel_noise_sigma*stdnoise)] = BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
+ "\n",
+ "fig = xana.heatmapPlot(np.log2(bad_pixels[:,:,0]),aspect=1, x_label='Column Number', y_label='Row Number', \n",
+ " lut_label='2^(Assigned Value to Bad Pixels)', x_range=(0,y), y_range=(0,x), \n",
+ " title = 'Bad Pixels Map Excluding Non-Sensitive Areas', panel_x_label= 'Columns Stat', \n",
+ " panel_y_label='Rows Stat')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Here, we are adding the pixels in the hole (center of the FastCCD) as well as 4 rows in the center of the detector, which we call overscan region:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def create_circular_mask(h, w, center=None, radius=None):\n",
+ "\n",
+ " import numpy as np\n",
+ " import math\n",
+ " \n",
+ " if center is None: # use the middle of the image\n",
+ " center = [int(w/2), int(h/2)]\n",
+ " if radius is None: # use the smallest distance between the center and image walls\n",
+ " radius = min(center[0], center[1], w-center[0], h-center[1])\n",
+ "\n",
+ " Y, X = np.ogrid[:h, :w]\n",
+ " dist_from_center = np.sqrt((X - center[0])**2 + (Y-center[1])**2)\n",
+ "\n",
+ " mask = dist_from_center < radius\n",
+ " return mask"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mask = np.zeros(offsetMap.shape, np.uint32)\n",
+ "\n",
+ "\n",
+ "# Defining a circular mask + a rectangular mask (overscan) for the hole in the middle of the CCD:\n",
+ "h, w = (x,y)\n",
+ "hole_mask_bool = create_circular_mask(h-4, w, radius=61.5, center=(w//2,(h-4)//2))\n",
+ "hole_mask = np.zeros(hole_mask_bool.shape, np.uint32)\n",
+ "hole_mask[hole_mask_bool] = BadPixels.NON_SENSITIVE.value\n",
+ "\n",
+ "overscan_mask = np.full((4, w), BadPixels.OVERSCAN.value) \n",
+ "\n",
+ "mask[:,:,0] = np.insert(hole_mask, (h-4)//2, overscan_mask, 0) \n",
+ "\n",
+ "# Assigning this masked area as bad pixels:\n",
+ "bad_pixels = np.bitwise_or(bad_pixels, mask)\n",
+ "fig = xana.heatmapPlot(np.log2(bad_pixels[:,:,0]),aspect=1, x_label='Column Number', y_label='Row Number', \n",
+ " lut_label='2^(Assigned Value to Bad Pixels)', x_range=(0,y), y_range=(0,x), panel_top_low_lim = 0, \n",
+ " panel_top_high_lim = 20, panel_side_low_lim = 0, panel_side_high_lim = 20, \n",
+ " title = 'Bad Pixels Map Including Non-Sensitive Areas', panel_x_label='Columns Stat', \n",
+ " panel_y_label='Rows Stat', vmax=20)\n",
+ "\n",
+ "#fig.savefig('BadPixelMap_1.svg', format='svg', dpi=1200, bbox_inches='tight') "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Third Iteration\n",
+ "\n",
+ "During the third iteration, the bad pixel map is applied to the data. Bad pixels are masked. Offset and common mode corrections are applied once again to the data, which now have bad pixdels excluded, to produce a common mode corrected noise map:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# bad_pixels is an array of (1934, 960, 1) filled with zeros except at indices where we have the actual bad pixels, whose\n",
+ "# values are set to be: 2 (2^1: BadPixels.OFFSET_OUT_OF_THRESHOLD.value), or\n",
+ "# 262144 (2^18: BadPixels.OVERSCAN.value), or 524288 (2^19: BadPixels.NON_SENSITIVE.value). These indices can be found\n",
+ "# using np.argwhere(bad_pixels != 0)\n",
+ "\n",
+ "event_threshold = sigmaNoise*np.median(noiseMapCM) # for exclusion of possible cosmic ray events\n",
+ "noiseCal.setBadPixelMask(bad_pixels != 0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for data in reader.readChunks():\n",
+ " data = data.astype(np.float32)\n",
+ " dx = np.count_nonzero(data, axis=(0, 1))\n",
+ " data = data[:,:,dx != 0]\n",
+ " data_copy = offsetCorrection.correct(copy.copy(data))\n",
+ " cellTable=np.zeros(data_copy.shape[2], np.int32)\n",
+ " data_copy = cmCorrection.correct(data_copy.astype(np.float32), cellTable=cellTable)\n",
+ " data[data_copy > event_threshold] = np.nan # cosmic rays\n",
+ " data = np.ma.MaskedArray(data, np.isnan(data), fill_value=0) # masking cosmics, the default fill_value is 1e+20 \n",
+ " data = offsetCorrection.correct(data)\n",
+ " offset_corr_data2 = copy.copy(data) # I am copying this so that I can have access to it in the table below\n",
+ " histCalCorrected.fill(data)\n",
+ " cellTable=np.zeros(data.shape[2], np.int32)\n",
+ " data = cmCorrection.correct(data.astype(np.float32), cellTable=cellTable)\n",
+ " histCalCMCorrected.fill(data)\n",
+ " noiseCal.fill(data) \n",
+ "\n",
+ "noiseMapCM_2nd = noiseCal.get().filled(0) # the masked pixels are filled with zero\n",
+ "ho2, eo2, co2, so2 = histCalCorrected.get()\n",
+ "hCM2, eCM2, cCM2 ,sCM2 = histCalCMCorrected.get()\n",
+ "print(\"Final iteration is Performed.\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Plots of the Final Results\n",
+ "\n",
+ "The following plot and table compare the offset and common mode corrected signal with and without the bad pixels:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "do_Final = [{'x': co_second_trial,\n",
+ " 'y': ho_second_trial,\n",
+ " 'y_err': np.sqrt(ho_second_trial[:]),\n",
+ " 'drawstyle': 'steps-mid',\n",
+ " 'color': 'blue',#'cornflowerblue',\n",
+ " 'errorstyle': 'bars',\n",
+ " 'label': 'Offset Corrected Signal, Bad Pixels Included - 2nd Trial'\n",
+ " },\n",
+ " {'x': cCM_second_trial,\n",
+ " 'y': hCM_second_trial,\n",
+ " 'y_err': np.sqrt(hCM_second_trial[:]),\n",
+ " 'drawstyle': 'steps-mid',\n",
+ " 'color': 'red',\n",
+ " 'errorstyle': 'bars',\n",
+ " 'ecolor': 'crimson',\n",
+ " 'label': 'Common Mode Corrected Signal, Bad Pixels Included - 2nd Trial' \n",
+ " },\n",
+ " {'x': co2,\n",
+ " 'y': ho2,\n",
+ " 'y_err': np.sqrt(ho2[:]),\n",
+ " 'drawstyle': 'steps-mid',\n",
+ " 'color': 'black', #'cornflowerblue',\n",
+ " 'errorstyle': 'bars',\n",
+ " 'label': 'Offset Corrected Signal, Bad Pixels Excluded - 3rd Trial'\n",
+ " },\n",
+ " {'x': cCM2,\n",
+ " 'y': hCM2,\n",
+ " 'y_err': np.sqrt(hCM2[:]),\n",
+ " 'drawstyle': 'steps-mid',\n",
+ " 'color': 'orange', #'cornflowerblue',\n",
+ " 'errorstyle': 'bars',\n",
+ " 'label': 'Common Mode Corrected Signal, Bad Pixels Excluded - 3rd Trial'\n",
+ " }]\n",
+ "\n",
+ "fig = xana.simplePlot(do_Final, figsize='2col', aspect=1, x_label = 'Corrected Signal (ADU)', \n",
+ " y_label=\"Counts (Logarithmic Scale)\", y_log=True, x_range=(-40,40), legend='bottom-left-frame-1col',\n",
+ " title = 'Comparison of Corrected Signal')\n",
+ "#fig.savefig('Corrected_Signal_Hist_2.svg', format='svg', dpi=1200, bbox_inches='tight') \n",
+ "\n",
+ "# offset_corr_data2 and data most likely have some nan's => I am going to use nanmean, nanmedian and nanstd functions:\n",
+ "t0 = PrettyTable()\n",
+ "t0.title = \"Comparison of the Second Round of Corrections - Bad Pixels Excluded\"\n",
+ "t0.field_names = [\"After Offset Correction\",\"After Common Mode Correction\"]\n",
+ "t0.add_row([\"Mean: {:0.3f} (ADU)\".format(np.nanmean(offset_corr_data2)), \"Mean: {:0.3f} (ADU)\".format(np.nanmean(data))])\n",
+ "t0.add_row([\"Median: {:0.3f} (ADU)\".format(np.nanmedian(offset_corr_data2)), \"Median: {:0.3f} (ADU)\".format(np.nanmedian(data))])\n",
+ "t0.add_row([\"Standard Deviation: {:0.3f} (ADU)\".format(np.nanstd(offset_corr_data2)), \"Standard Deviation: {:0.3f} (ADU)\".format(np.nanstd(data))])\n",
+ "print(t0,'\\n')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Final NoiseMap\n",
+ "\n",
+ "The effect of exclusion of bad pixels on common mode corrected noise is shown below. Finally common mode corrected noise map with bad pixels excluded (noiseMapCM_2nd) is displayed:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "#*****NOISE MAP HISTOGRAM FROM THE COMMON MODE CORRECTED DATA*******#\n",
+ "hn_CM2,cn_CM2 = np.histogram(noiseMapCM_2nd.flatten(), bins=200, range=(2,40))\n",
+ "\n",
+ "dn2 = [{'x': cn[:-1],\n",
+ " 'y': hn,\n",
+ " #'y_err': np.sqrt(hn[:]),\n",
+ " 'drawstyle': 'steps-mid',#'bars',\n",
+ " 'color': 'blue', #'cornflowerblue',\n",
+ " 'label': 'Uncorrected Noise'\n",
+ " },\n",
+ " {'x': cn_CM[:-1],\n",
+ " 'y': hn_CM,\n",
+ " #'y_err': np.sqrt(hn_CM[:]),\n",
+ " 'drawstyle': 'steps-mid',\n",
+ " 'color': 'red',\n",
+ " #'ecolor': 'crimson',\n",
+ " 'label': 'Common Mode Corrected Noise prior to Bad Pixels Exclusion'\n",
+ " },\n",
+ " {'x': cn_CM2[:-1],\n",
+ " 'y': hn_CM2,\n",
+ " #'y_err': np.sqrt(hn_CM2[:]),\n",
+ " 'drawstyle': 'steps-mid',\n",
+ " 'color': 'black', #'cornflowerblue',\n",
+ " 'label': 'Common Mode Corrected Noise after Bad Pixels Exclusion'\n",
+ " }]\n",
+ "\n",
+ "fig = xana.simplePlot(dn2, figsize='2col', aspect = 1, x_label = 'Noise (ADU)', y_label=\"Counts\", y_log=True, \n",
+ " x_range=(0,40), y_range=(0,1e6), legend='top-right-frame-1col', title = 'Final Noise Comparison')\n",
+ "\n",
+ "#fig.savefig('Noise_Hist_2.svg', format='svg', dpi=1200, bbox_inches='tight') \n",
+ "\n",
+ "fig = xana.heatmapPlot(np.log2(noiseMapCM_2nd[:,:,0]), aspect=1, x_label='Column Number', y_label='Row Number',\n",
+ " lut_label='Noise (ADU)', x_range=(0,y), y_range=(0,x), vmax=2*np.mean(noiseMapCM_2nd), \n",
+ " title = 'Final Common Mode Corrected Noise (Bad Pixels Excluded)', \n",
+ " panel_x_label='Columns Stat (ADU)', panel_y_label='Rows Stat (ADU)')\n",
+ "#fig.savefig('NoiseMapCM_2nd.pdf', format='pdf', dpi=400, bbox_inches='tight') "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Final Bad Pixel Map\n",
+ "\n",
+ "Lastly, the final bad pixel map is generated based on the OffsetMap and the noiseMapCM_2nd (common mode corrected noise after exclusion of the initial bad pixels):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "bad_pixels = np.zeros(offsetMap.shape, np.uint32)\n",
+ "mnoffset = np.nanmedian(offsetMap)\n",
+ "stdoffset = np.nanstd(offsetMap)\n",
+ "bad_pixels[(offsetMap < mnoffset-bad_pixel_offset_sigma*stdoffset) | \n",
+ " (offsetMap > mnoffset+bad_pixel_offset_sigma*stdoffset)] = BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
+ "\n",
+ "mnnoise = np.nanmedian(noiseMapCM_2nd)\n",
+ "stdnoise = np.nanstd(noiseMapCM_2nd)\n",
+ "bad_pixels[(noiseMapCM_2nd < mnnoise-bad_pixel_noise_sigma*stdnoise) | \n",
+ " (noiseMapCM_2nd > mnnoise+bad_pixel_noise_sigma*stdnoise)] = BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
+ "\n",
+ "bad_pixels = np.bitwise_or(bad_pixels, mask)\n",
+ "fig = xana.heatmapPlot(np.log2(bad_pixels[:,:,0]),aspect=1, x_label='Column Number', y_label='Row Number', \n",
+ " lut_label='2^(Assigned Value to Bad Pixels)', x_range=(0,y), y_range=(0,x), panel_top_low_lim = 0, \n",
+ " panel_top_high_lim = 20, panel_side_low_lim = 0, panel_side_high_lim = 20, \n",
+ " title = 'Final Bad Pixels Map', panel_x_label='Columns Stat', \n",
+ " panel_y_label='Rows Stat', vmax=20)\n",
+ "#fig.savefig('BadPixelMap_2.svg', format='svg', dpi=1200, bbox_inches='tight') "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown('### Statistics on the Bad Pixels'))\n",
+ "num_bad_pixels = np.count_nonzero(bad_pixels)\n",
+ "num_all_pixels = x*y\n",
+ "percentage_bad_pixels = num_bad_pixels*100/num_all_pixels\n",
+ "print(\"Number of bad pixels: {:0.0f}, i.e. {:0.2f}% of all pixels\".format(num_bad_pixels, percentage_bad_pixels))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Electronic Noise\n",
+ "\n",
+ "According to Table 6.1 (page 80) of Ivana KlaÄková's master's thesis: \"Conversion gain for the FastCCD is: lower hemisphere = 6.2e-/ADU and upper hemisphere = 6.1e-/ADU.\"\n",
+ "\n",
+ "The following Tables present the noise along lower hemisphere, upper hemisphere, and the entire FastCCD detector at different stages. Here, the values in the first table (in ADU and e-) are the mean of noise per pixel, where noise is considered to be the initial uncorrected noise, CM corrected noise after second trial (including bad pixels) and CM corrected noise after third trial (excluding bad pixels). \n",
+ "\n",
+ "The values of the second table (in electrons) are the standard deviation of noise per pixel."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# noiseMap refers to the initial uncorrected noise, noiseMapCM refers to common mode corrected noise with inclusion of \n",
+ "# bad pixels, and noiseMapCM_2nd refers to common mode corrected noise without inclusion of bad pixels:\n",
+ "\n",
+ "ADU_to_electron = (ADU_to_electron_upper + ADU_to_electron_lower)/2 # Average of ADU_to_electron for the entire detector \n",
+ "\n",
+ "print(\"Abbreviations:\")\n",
+ "print(\" - ED = Entire Detector; LH: Lower Hemisphere; UH: Upper Hemisphere\")\n",
+ "print(\" - CM Noise: Common Mode Corrected Noise\")\n",
+ "print(\" - BP: Bad Pixels\\n\")\n",
+ " \n",
+ "t0 = PrettyTable()\n",
+ "t0.title = \"Averages of Noise per Pixel\"\n",
+ "t0.field_names = [\"Uncorrected Noise\",\"CM Noise, BP Incl.\", \"CM Noise, BP Excl.\"]\n",
+ "t0.add_row([\"ED: {:0.2f} ADU = {:0.2f} e-\".format(np.mean(noiseMap),np.mean(noiseMap)*ADU_to_electron), \"ED: {:0.2f} ADU = {:0.2f} e-\".format(np.mean(noiseMapCM), np.mean(noiseMapCM)*ADU_to_electron), \"ED: {:0.2f} ADU = {:0.2f} e-\".format(np.mean(noiseMapCM_2nd), np.mean(noiseMapCM_2nd)*ADU_to_electron)])\n",
+ "t0.add_row([\"LH: {:0.2f} ADU = {:0.2f} e-\".format(np.mean(noiseMap[:x//2,:]), np.mean(noiseMap[:x//2,:])*ADU_to_electron_lower), \"LH: {:0.2f} ADU = {:0.2f} e-\".format(np.mean(noiseMapCM[:x//2,:]), np.mean(noiseMapCM[:x//2,:])*ADU_to_electron_lower), \"LH: {:0.2f} ADU = {:0.2f} e-\".format(np.mean(noiseMapCM_2nd[:x//2,:]), np.mean(noiseMapCM_2nd[:x//2,:])*ADU_to_electron_lower)])\n",
+ "t0.add_row([\"UH: {:0.2f} ADU = {:0.2f} e-\".format(np.mean(noiseMap[x//2:,:]), np.mean(noiseMap[x//2:,:])*ADU_to_electron_upper), \"UH: {:0.2f} ADU = {:0.2f} e-\".format(np.mean(noiseMapCM[x//2:,:]), np.mean(noiseMapCM[x//2:,:])*ADU_to_electron_upper), \"UH: {:0.2f} ADU = {:0.2f} e-\".format(np.mean(noiseMapCM_2nd[x//2:,:]), np.mean(noiseMapCM_2nd[x//2:,:])*ADU_to_electron_upper)])\n",
+ "print(t0,'\\n')\n",
+ "\n",
+ "t1 = PrettyTable()\n",
+ "t1.title = \"Standard Deviations of Noise per Pixel\"\n",
+ "t1.field_names = [\"Uncorrected Noise\",\"CM Noise, BP Incl.\", \"CM Noise, BP Excl.\"]\n",
+ "t1.add_row([\"ED: {:0.2f} e-\".format(np.std(noiseMap)*ADU_to_electron), \"ED: {:0.2f} e-\".format(np.std(noiseMapCM)*ADU_to_electron), \"ED: {:0.2f} e-\".format(np.std(noiseMapCM_2nd)*ADU_to_electron)])\n",
+ "t1.add_row([\"LH: {:0.2f} e-\".format(np.std(noiseMap[:x//2,:])*ADU_to_electron_lower), \"LH: {:0.2f} e-\".format(np.std(noiseMapCM[:x//2,:])*ADU_to_electron_lower), \"LH: {:0.2f} e-\".format(np.std(noiseMapCM_2nd[:x//2,:])*ADU_to_electron_lower)])\n",
+ "t1.add_row([\"UH: {:0.2f} e-\".format(np.std(noiseMap[x//2:,:])*ADU_to_electron_upper), \"UH: {:0.2f} e-\".format(np.std(noiseMapCM[x//2:,:])*ADU_to_electron_upper), \"UH: {:0.2f} e-\".format(np.std(noiseMapCM_2nd[x//2:,:])*ADU_to_electron_upper)])\n",
+ "print(t1)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Calibration Constants"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dictionary = {} \n",
+ "dictionary['Offset'] = offsetMap.data\n",
+ "dictionary['Noise'] = noiseMapCM_2nd.data\n",
+ "dictionary['BadPixelsDark'] = bad_pixels.data\n",
+ "\n",
+ "for const in dictionary:\n",
+ " metadata = ConstantMetaData()\n",
+ " dconst = getattr(Constants.CCD(DetectorTypes.fastCCD), const)()\n",
+ " dconst.data = dictionary[const]\n",
+ " metadata.calibration_constant = dconst\n",
+ " \n",
+ " condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
+ " integration_time=integration_time,\n",
+ " gain_setting=det_gain,\n",
+ " temperature=temperature_k,\n",
+ " pixels_x=1934,\n",
+ " pixels_y=960)\n",
+ " \n",
+ " for parm in condition.parameters:\n",
+ " if parm.name == \"Sensor Temperature\":\n",
+ " parm.lower_deviation = temp_limits\n",
+ " parm.upper_deviation = temp_limits\n",
+ "\n",
+ " device = Detectors.fastCCD1\n",
+ " metadata.detector_condition = condition\n",
+ " \n",
+ " # Specifying the a version for this constant:\n",
+ " if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
+ " \n",
+ " if db_output:\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout) \n",
+ " \n",
+ "print(\"Calibration constants (offsetMap, noiseMapCM_2nd and bad_pixels) are sent to the calibration database.\")\n",
+ "print(\"Creation time is: {}\".format(creation_time))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.7"
+ },
+ "latex_envs": {
+ "LaTeX_envs_menu_present": true,
+ "autocomplete": true,
+ "bibliofile": "biblio.bib",
+ "cite_by": "apalike",
+ "current_citInitial": 1,
+ "eqLabelWithNumbers": true,
+ "eqNumInitial": 1,
+ "hotkeys": {
+ "equation": "Ctrl-E",
+ "itemize": "Ctrl-I"
+ },
+ "labels_anchors": false,
+ "latex_user_defs": false,
+ "report_style_numbering": false,
+ "user_envs_cfg": false
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index 3d68bf29285488831320aa526f96e2c027c6d6ec..5e7badc9f57e37c9b7ef279d60ba8f9da0bbf469 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -117,7 +117,7 @@ notebooks = {
},
"FASTCCD": {
"DARK": {
- "notebook": "notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC.ipynb",
+ "notebook": "notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb",
"concurrency": {"parameter": None,
"default concurrency": None,
"cluster cores": 16},