diff --git a/notebooks/ePix100/Characterize_FlatFields_ePix100_NBC.ipynb b/notebooks/ePix100/Characterize_FlatFields_ePix100_NBC.ipynb
index 9b036373ec3ef2a0060d5910dc7f288669204f0c..12b279e6bee02c9395567b88840f3947d4795161 100644
--- a/notebooks/ePix100/Characterize_FlatFields_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Characterize_FlatFields_ePix100_NBC.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "markdown",
- "id": "b195739b",
+ "id": "826b869a",
"metadata": {},
"source": [
"# ePix100 Flat Field Characterization\n",
@@ -15,7 +15,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "456b704f",
+ "id": "7439b810",
"metadata": {},
"outputs": [],
"source": [
@@ -29,9 +29,7 @@
"karabo_id = \"MID_EXP_EPIX-2\" # karabo karabo_id\n",
"karabo_da = \"EPIX02\" # data aggregators\n",
"receiver_template = \"RECEIVER\" # detector receiver template for accessing raw data files\n",
- "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
"instrument_source_template = '{}/DET/{}:daqOutput' # instrument detector data source in h5files\n",
- "constants_path = '' # Use constants in given constant file path\n",
"\n",
"# Fit parameters\n",
"peak_fitting = 'gauss' # method to find the peak position per pixel: 'median' or 'gauss'\n",
@@ -47,9 +45,9 @@
"split_evt_mip_threshold = 1000 # Threshold for rejection of MIP events (e.g, cosmic-rays)\n",
"\n",
"# Parameters for the calibration database.\n",
- "use_dir_creation_date = True\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
"cal_db_timeout = 300000 # timeout on caldb requests\n",
+ "creation_time = \"\" # The timestamp to use with Calibration DB. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
"db_output = False # Output constants to the calibration database\n",
"local_output = True # Output constants locally\n",
"\n",
@@ -62,24 +60,16 @@
"\n",
"# Parameters used during selecting raw data trains.\n",
"min_trains = 1 # Minimum number of trains that should be available. Default 1.\n",
- "max_trains = 0 # Maximum number of trains to use for processing. Set to 0 to use all available trains.\n",
- "\n",
- "# Don't delete! myMDC sends this by default.\n",
- "operation_mode = '' # Detector operation mode, optional\n",
- "\n",
- "# TODO: delete after removing from calibration_configurations\n",
- "db_module = '' # ID of module in calibration database, this parameter is ignored in the notebook. TODO: remove from calibration_configurations."
+ "max_trains = 0 # Maximum number of trains to use for processing. Set to 0 to use all available trains."
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "eed2b98e",
+ "id": "43791d97",
"metadata": {},
"outputs": [],
"source": [
- "import os\n",
- "import time\n",
"import warnings\n",
"\n",
"import matplotlib.pyplot as plt\n",
@@ -100,6 +90,7 @@
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.epix100 import epix100lib\n",
"from cal_tools.tools import (\n",
+ " calcat_creation_time,\n",
" get_dir_creation_date,\n",
" get_pdu_from_db,\n",
" get_constant_from_db,\n",
@@ -107,13 +98,13 @@
" save_const_to_h5,\n",
" send_to_db,\n",
")\n",
- "from iCalibrationDB import Conditions,Constants,Detectors"
+ "from iCalibrationDB import Conditions, Constants, Detectors"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "6db90bbe",
+ "id": "4f4d9f62",
"metadata": {},
"outputs": [],
"source": [
@@ -131,7 +122,7 @@
},
{
"cell_type": "markdown",
- "id": "282bf2f5",
+ "id": "6571ae1c",
"metadata": {},
"source": [
"## Load Data"
@@ -140,7 +131,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "a7004c0c",
+ "id": "9c93190b",
"metadata": {},
"outputs": [],
"source": [
@@ -157,17 +148,14 @@
"print(f\"Run is: {run}\")\n",
"print(f\"Instrument H5File source: {instrument_src}\")\n",
"\n",
- "creation_time = None\n",
- "if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run)\n",
- "if creation_time:\n",
- " print(f\"Using {creation_time.isoformat()} as creation time\")"
+ "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
+ "print(f\"Using {creation_time.isoformat()} as creation time\")"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "9d75f4b7",
+ "id": "08d6fef2",
"metadata": {
"slideshow": {
"slide_type": "-"
@@ -191,8 +179,7 @@
" \" Not enough data to process flat-fields.\")\n",
"\n",
"# Sequences to read\n",
- "seq_files = [Path(f.filename) for f in run_dir.select(f'*{karabo_id}*').files]\n",
- "seq_files = sorted(seq_files)\n",
+ "seq_files = sorted([Path(f.filename) for f in run_dir.select(f'*{karabo_id}*').files])\n",
"seq0_size = H5File(seq_files[0]).get_data_counts(*pixels_src).size\n",
"\n",
"if sequences != [-1]:\n",
@@ -204,27 +191,35 @@
" raise IndexError(\"No sequence files available for the selected sequences.\")\n",
"\n",
"# Trains to be processed\n",
- "n_trains = run_dir.get_data_counts(*pixels_src).shape[0]\n",
- "dshape = run_dir.select(*pixels_src)[pixels_src].shape\n",
+ "trains = np.ndarray(0,dtype=int)\n",
+ "for seq in seq_files:\n",
+ " seq_str = str(seq)\n",
+ " H5File(seq).get_data_counts(*pixels_src).size\n",
+ " seq_size = H5File(seq).get_data_counts(*pixels_src).size\n",
+ " n = int(seq_str[seq_str.rfind('-S0')+len('-S0'):seq_str.rfind('.h5')])\n",
+ " t = np.arange(n*seq0_size,n*seq0_size+seq_size)\n",
+ " trains = np.append(trains,t)\n",
+ " \n",
+ "n_trains = run_dir.select_trains(trains).get_data_counts(*pixels_src).shape[0]\n",
+ "dshape = run_dir.select_trains(trains).select(*pixels_src)[pixels_src].shape\n",
"\n",
"if n_trains != dshape[0]:\n",
" print(f\"Warning: {n_trains - dshape[0]} trains with empty data.\")\n",
" n_trains = dshape[0]\n",
"\n",
- "trains = np.arange(0,n_trains)\n",
- "data_dc = run_dir.select(*pixels_src,require_all=True).select_trains(trains)\n",
+ "trains = trains[:n_trains]\n",
"\n",
"print(f\"Reading from: \")\n",
"[print(f'\\t{seq}') for seq in seq_files]\n",
"print('\\nAvailable sequece files: ' + str(len(run_dir.select(f'*{karabo_id}*').files)))\n",
"print(f'Sequence files used for processing: {len(seq_files)}')\n",
- "print(f'Images to analyze: {trains.size}')"
+ "print(f'Images to analyze: {n_trains}')"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "dded1e6e",
+ "id": "9fdf1715",
"metadata": {},
"outputs": [],
"source": [
@@ -266,7 +261,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "893871d3",
+ "id": "a4dd3d8d",
"metadata": {},
"outputs": [],
"source": [
@@ -282,7 +277,7 @@
},
{
"cell_type": "markdown",
- "id": "d2a1a89f",
+ "id": "7920cb0b",
"metadata": {
"tags": []
},
@@ -293,12 +288,12 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "1c0cd3f5",
+ "id": "593964be",
"metadata": {},
"outputs": [],
"source": [
"const_data = dict()\n",
- "constants = ['Offset','Noise', 'BadPixelsDark']\n",
+ "constants = ['Offset', 'Noise', 'BadPixelsDark']\n",
"\n",
"condition = Conditions.Dark.ePix100(bias_voltage=bias_voltage,\n",
" integration_time=integration_time,\n",
@@ -321,7 +316,7 @@
},
{
"cell_type": "markdown",
- "id": "d6e5ea6e",
+ "id": "ea05e961",
"metadata": {},
"source": [
"## Instantiate calculators"
@@ -330,7 +325,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "e45cca09",
+ "id": "f05e8297",
"metadata": {},
"outputs": [],
"source": [
@@ -379,7 +374,7 @@
},
{
"cell_type": "markdown",
- "id": "dba97bf9",
+ "id": "8977c9ff",
"metadata": {},
"source": [
"## Correct data"
@@ -388,12 +383,12 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "b3c46b59",
+ "id": "de145b05",
"metadata": {},
"outputs": [],
"source": [
- "bin_min = -50\n",
- "bin_max = 800\n",
+ "bin_min = ADU_range[0]\n",
+ "bin_max = ADU_range[1]\n",
"bin_width = 1\n",
"\n",
"bins = np.arange(bin_min,bin_max,bin_width)\n",
@@ -403,7 +398,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "4447227d",
+ "id": "1765d3f8",
"metadata": {},
"outputs": [],
"source": [
@@ -431,14 +426,14 @@
" hist['CS'] += np.histogram(d[d>0].flatten(),bins=bins)[0]\n",
" hist['S'] += np.histogram(sing[sing>0].flatten(),bins=bins)[0]\n",
" \n",
- " data_corr[index+prev_chunk] = d\n",
+ " [index+prev_chunk] = d\n",
" data_singles[index+prev_chunk] = sing.swapaxes(0,-1)"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "36a952e2",
+ "id": "7c4dcd5b",
"metadata": {},
"outputs": [],
"source": [
@@ -467,7 +462,7 @@
},
{
"cell_type": "markdown",
- "id": "19fa85ec",
+ "id": "923a5ac4",
"metadata": {},
"source": [
"## Plot histograms"
@@ -476,7 +471,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "1862b145",
+ "id": "c43ae1dd",
"metadata": {},
"outputs": [],
"source": [
@@ -498,7 +493,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "1c6f3528",
+ "id": "46d9fe80",
"metadata": {},
"outputs": [],
"source": [
@@ -528,7 +523,7 @@
},
{
"cell_type": "markdown",
- "id": "74416dc3",
+ "id": "7739d666",
"metadata": {},
"source": [
"## Flat-Field Statistics"
@@ -537,7 +532,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "9c1c3b53",
+ "id": "ed100e6a",
"metadata": {},
"outputs": [],
"source": [
@@ -555,7 +550,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "721f774d",
+ "id": "a649666b",
"metadata": {},
"outputs": [],
"source": [
@@ -595,7 +590,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "9fd2f808",
+ "id": "d4bba07d",
"metadata": {},
"outputs": [],
"source": [
@@ -614,7 +609,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "9d58e028",
+ "id": "c5986694",
"metadata": {
"scrolled": false
},
@@ -649,7 +644,7 @@
},
{
"cell_type": "markdown",
- "id": "3624621d",
+ "id": "3d58fc82",
"metadata": {},
"source": [
"## Plot random sample pixels "
@@ -658,7 +653,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "ddb16296",
+ "id": "62b2650e",
"metadata": {},
"outputs": [],
"source": [
@@ -708,7 +703,7 @@
},
{
"cell_type": "markdown",
- "id": "928f899c",
+ "id": "a968c8df",
"metadata": {},
"source": [
"## Fit single photon peaks per pixel"
@@ -717,7 +712,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "2a4bccbe",
+ "id": "49d52f2b",
"metadata": {},
"outputs": [],
"source": [
@@ -748,7 +743,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "9518f337",
+ "id": "8891bcd4",
"metadata": {},
"outputs": [],
"source": [
@@ -791,7 +786,7 @@
},
{
"cell_type": "markdown",
- "id": "4d8f8441",
+ "id": "e17b27ef",
"metadata": {},
"source": [
"## Flat-Field Bad Pixels"
@@ -800,7 +795,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "ee754f67",
+ "id": "0816af0f",
"metadata": {},
"outputs": [],
"source": [
@@ -847,7 +842,7 @@
},
{
"cell_type": "markdown",
- "id": "7c6aba1b",
+ "id": "e97610e2",
"metadata": {},
"source": [
"## Relative Gain Map"
@@ -856,7 +851,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "e0fbcaa8",
+ "id": "1ea03d36",
"metadata": {},
"outputs": [],
"source": [
@@ -880,7 +875,7 @@
},
{
"cell_type": "markdown",
- "id": "18fe5723",
+ "id": "c2870edc",
"metadata": {},
"source": [
"## Absolute Gain Conversion Constant"
@@ -889,7 +884,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "4a4882fb",
+ "id": "282ad58a",
"metadata": {},
"outputs": [],
"source": [
@@ -915,7 +910,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "0db79cd0",
+ "id": "3a0daabf",
"metadata": {},
"outputs": [],
"source": [
@@ -951,7 +946,7 @@
},
{
"cell_type": "markdown",
- "id": "9a6bb012",
+ "id": "c93fb9ac",
"metadata": {},
"source": [
"## Gain Map Validation\n",
@@ -964,7 +959,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "8ef93ce7",
+ "id": "8792ff72",
"metadata": {},
"outputs": [],
"source": [
@@ -997,7 +992,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "7be3645c",
+ "id": "1150be55",
"metadata": {},
"outputs": [],
"source": [
@@ -1028,7 +1023,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "1db20c6a",
+ "id": "e55aa651",
"metadata": {},
"outputs": [],
"source": [
@@ -1066,7 +1061,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "4571dfee",
+ "id": "a1319015",
"metadata": {},
"outputs": [],
"source": [
@@ -1088,7 +1083,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "ea1f0175",
+ "id": "20f9faa5",
"metadata": {},
"outputs": [],
"source": [
@@ -1123,7 +1118,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "86882291",
+ "id": "c35bddec",
"metadata": {
"scrolled": false
},
@@ -1226,7 +1221,7 @@
},
{
"cell_type": "markdown",
- "id": "1f796a91",
+ "id": "205f27b9",
"metadata": {},
"source": [
"## Linearity Analysis"
@@ -1235,7 +1230,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "0033326a",
+ "id": "d6cf1264",
"metadata": {},
"outputs": [],
"source": [
@@ -1295,7 +1290,7 @@
},
{
"cell_type": "markdown",
- "id": "6c45b833",
+ "id": "441e426a",
"metadata": {},
"source": [
"## Energy Resolution Analysis"
@@ -1304,7 +1299,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "f9e1ea35",
+ "id": "feb7a5bf",
"metadata": {},
"outputs": [],
"source": [
@@ -1318,7 +1313,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "c5451f6a",
+ "id": "25b3f89a",
"metadata": {},
"outputs": [],
"source": [
@@ -1348,7 +1343,7 @@
},
{
"cell_type": "markdown",
- "id": "dd6572f5",
+ "id": "f85f601d",
"metadata": {},
"source": [
"## Calibration Constants DB\n",
@@ -1358,7 +1353,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "452c9581",
+ "id": "b898799f",
"metadata": {},
"outputs": [],
"source": [