diff --git a/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb b/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
index 7f4b500b3cdbcb40cc4149e1f51f4fbbbf1011a8..3d9032582b7d2da7f2e1fc33c9cefa49eb675ebe 100644
--- a/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
+++ b/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
@@ -21,21 +21,21 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
"start_date = \"2019-01-01\" # Date to start investigation interval from\n",
- "end_date = \"2019-12-12\" # Date to end investigation interval at, can be \"now\"\n",
+ "end_date = \"NOW\" # Date to end investigation interval at, can be \"now\"\n",
"nconstants = 20 # Number of time stamps to plot. If not 0, overcome start_date.\n",
- "constants = [\"Noise\", \"SlopesFF\", \"SlopesPC\", \"Offset\"] # Constants to plot\n",
+ "constants = [\"Noise\", \"Offset\", \"SlopesFF\", \"SlopesPC\"] # Constants to plot\n",
"modules = [1] # Modules, set to -1 for all, range allowed\n",
- "bias_voltages = [300, 500] # Bias voltage\n",
- "mem_cells = [128, 176, 202, 250] # Number of used memory cells. Typically: 4,32,64,128,176.\n",
+ "bias_voltages = [300] # Bias voltage\n",
+ "mem_cells = [250] # Number of used memory cells. Typically: 4,32,64,128,176.\n",
"acquisition_rate = [0.0, 1.1, 2.2, 4.5]\n",
"photon_energy = 9.2 # Photon energy of the beam\n",
- "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_AGIPD/\" # Output folder, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_AGIPD55/\" # Output folder, required\n",
"use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
"cal_db_timeout = 120000 # timeout on caldb requests\",\n",
"adu_to_photon = 33.17 # ADU to photon conversion factor (8000 / 3.6 / 67.)\n",
@@ -50,12 +50,13 @@
"range_noise_e = [85., 500., 85., 500.] # plotting range for noise in [e-]: high gain l, r, medium gain l, r \n",
"range_slopesPC = [22.0, 27.0, -0.5, 1.5] # plotting range for slope PC: high gain l, r, medium gain l, r \n",
"range_slopesFF = [0.8, 1.2, 0.6, 1.2] # plotting range for slope FF: high gain l, r, medium gain l, r \n",
- "plot_range = 3 # range for plotting in units of median absolute deviations"
+ "plot_range = 3 # range for plotting in units of median absolute deviations\n",
+ "x_labels = ['Acquisition rate', 'Memory cells'] # parameters to be shown on X axis"
]
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"metadata": {
"scrolled": true
},
@@ -79,21 +80,9 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Bad pixels data: {'SlopesFF': 'BadPixelsFF', 'SlopesPC': 'BadPixelsPC', 'Noise': 'BadPixelsDark', 'Offset': 'BadPixelsDark'}\n",
- "CalDB Interface: tcp://max-exfl016:8015#8025\n",
- "Start time at: 2019-01-01 00:00:00\n",
- "End time at: 2019-12-12 00:00:00\n",
- "Modules: ['Q1M2']\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"# Prepare variables\n",
"nMem = max(mem_cells) # Number of mem Cells to store\n",
@@ -133,17 +122,9 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "[{'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': None}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 1.1}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 2.2}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 4.5}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': None}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': 1.1}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': 2.2}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': 4.5}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': None}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': 1.1}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': 2.2}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': 4.5}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': None}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': 1.1}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': 2.2}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': 4.5}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': None}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 1.1}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 2.2}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 4.5}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': None}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': 1.1}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': 2.2}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': 4.5}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': None}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': 1.1}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': 2.2}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': 4.5}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': None}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': 1.1}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': 2.2}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': 4.5}]\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"parameter_list = combine_lists(bias_voltages, modules, mem_cells, acquisition_rate,\n",
" names = ['bias_voltage', 'module', 'mem_cells', 'acquisition_rate'])\n",
@@ -152,91 +133,11 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": null,
"metadata": {
"scrolled": false
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Request: Noise with paramters: {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': None}\n",
- "constantDark: BadPixelsDark\n",
- "Item: {'id': 6990, 'name': '20190220_072419_sIdx=0', 'file_name': 'cal.1550647458.8201447.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2019-02-19T15:54:20.000+01:00', 'end_validity_at': None, 'begin_at': '2019-02-19T15:54:20.000+01:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2019-02-19 15:54:20+01:00\n",
- "Found bad pixels at 2019-02-19 15:54:20+01:00\n",
- "Item: {'id': 5471, 'name': '20181027_075853_sIdx=0', 'file_name': 'cal.1540627132.6379695.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-10-27T09:20:58.000+02:00', 'end_validity_at': None, 'begin_at': '2018-10-27T09:20:58.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2018-10-27 09:20:58+02:00\n",
- "Found bad pixels at 2018-10-27 09:20:58+02:00\n",
- "Item: {'id': 6791, 'name': '20181213_160158_sIdx=0', 'file_name': 'cal.1544716917.4122818.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-10-22T22:24:16.000+02:00', 'end_validity_at': None, 'begin_at': '2018-10-22T22:24:16.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2018-10-22 22:24:16+02:00\n",
- "Found bad pixels at 2018-10-22 22:24:16+02:00\n",
- "Item: {'id': 6834, 'name': '20190207_135120_sIdx=0', 'file_name': 'cal.1549547479.2884257.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-10-21T20:55:02.000+02:00', 'end_validity_at': None, 'begin_at': '2018-10-21T20:55:02.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2018-10-21 20:55:02+02:00\n",
- "Found bad pixels at 2018-10-21 20:55:02+02:00\n",
- "Item: {'id': 4670, 'name': '20180923_021047_sIdx=0', 'file_name': 'cal.1537668646.6458402.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-09-23T04:10:45.000+02:00', 'end_validity_at': None, 'begin_at': '2018-09-23T04:10:45.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2018-09-23 04:10:45+02:00\n",
- "Found bad pixels at 2018-09-23 04:10:50+02:00\n",
- "Item: {'id': 4546, 'name': '20180923_000251_sIdx=0', 'file_name': 'cal.1537660970.398414.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-09-23T02:02:49.000+02:00', 'end_validity_at': None, 'begin_at': '2018-09-23T02:02:49.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2018-09-23 02:02:49+02:00\n",
- "Found bad pixels at 2018-09-23 02:02:57+02:00\n",
- "Item: {'id': 4482, 'name': '20180922_235703_sIdx=0', 'file_name': 'cal.1537660622.794079.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-09-23T01:57:01.000+02:00', 'end_validity_at': None, 'begin_at': '2018-09-23T01:57:01.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2018-09-23 01:57:01+02:00\n",
- "Found bad pixels at 2018-09-23 01:57:06+02:00\n",
- "Item: {'id': 5269, 'name': '20181025_061038_sIdx=0', 'file_name': 'cal.1540447837.275528.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-09-02T16:25:41.000+02:00', 'end_validity_at': None, 'begin_at': '2018-09-02T16:25:41.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2018-09-02 16:25:41+02:00\n",
- "Found bad pixels at 2018-09-02 16:25:41+02:00\n",
- "Item: {'id': 3431, 'name': '20180901_173920_sIdx=0', 'file_name': 'cal.1535823559.5818477.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-09-01T19:39:13.000+02:00', 'end_validity_at': None, 'begin_at': '2018-09-01T19:39:13.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2018-09-01 19:39:13+02:00\n",
- "Found bad pixels at 2018-09-01 19:39:29+02:00\n",
- "Item: {'id': 3229, 'name': '20180830_164924_sIdx=0', 'file_name': 'cal.1535647763.2838182.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-08-30T18:49:21.000+02:00', 'end_validity_at': None, 'begin_at': '2018-08-30T18:49:21.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2018-08-30 18:49:21+02:00\n",
- "Found bad pixels at 2018-08-30 18:49:26+02:00\n",
- "Item: {'id': 6413, 'name': '20181210_082033_sIdx=0', 'file_name': 'cal.1544430032.7715538.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-01-29T17:10:23.000+01:00', 'end_validity_at': None, 'begin_at': '2018-01-29T17:10:23.000+01:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2018-01-29 17:10:23+01:00\n",
- "Found bad pixels at 2018-01-29 17:10:23+01:00\n",
- "Item: {'id': 6348, 'name': '20181210_081739_sIdx=0', 'file_name': 'cal.1544429858.2109814.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2017-12-04T20:15:40.000+01:00', 'end_validity_at': None, 'begin_at': '2017-12-04T20:15:40.000+01:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Bad pixels are not found!\n",
- "Item: {'id': 6590, 'name': '20181210_092654_sIdx=0', 'file_name': 'cal.1544434013.2191806.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2017-10-10T19:00:34.000+02:00', 'end_validity_at': None, 'begin_at': '2017-10-10T19:00:34.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2017-10-10 19:00:34+02:00\n",
- "Found bad pixels at 2017-10-10 19:00:34+02:00\n",
- "Item: {'id': 6509, 'name': '20181210_085802_sIdx=0', 'file_name': 'cal.1544432281.1941464.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2017-09-19T15:10:49.000+02:00', 'end_validity_at': None, 'begin_at': '2017-09-19T15:10:49.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2017-09-19 15:10:49+02:00\n",
- "Found bad pixels at 2017-09-19 15:10:49+02:00\n",
- "Item: {'id': 6543, 'name': '20181210_085904_sIdx=0', 'file_name': 'cal.1544432342.903418.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2017-09-15T23:33:29.000+02:00', 'end_validity_at': None, 'begin_at': '2017-09-15T23:33:29.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
- "Found constant Noise: begin at 2017-09-15 23:33:29+02:00\n",
- "Found bad pixels at 2017-09-15 23:33:29+02:00\n",
- "Request: Noise with paramters: {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 1.1}\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "constantDark: BadPixelsDark\n",
- "Request: Noise with paramters: {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 2.2}\n"
- ]
- },
- {
- "ename": "KeyboardInterrupt",
- "evalue": "",
- "output_type": "error",
- "traceback": [
- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
- "\u001b[0;32m<ipython-input-5-a602d73b9ab9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcal_db_timeout\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[0mmeta_only\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m version_info=True)\n\u001b[0m\u001b[1;32m 41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[0;31m# Request BP constant versions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;32m/gpfs/exfel/data/scratch/karnem/calibration3/pycalibration/cal_tools/cal_tools/tools.py\u001b[0m in \u001b[0;36mget_from_db\u001b[0;34m(device, constant, condition, empty_constant, cal_db_interface, creation_time, verbosity, timeout, ntries, meta_only, version_info)\u001b[0m\n\u001b[1;32m 512\u001b[0m r = metadata.retrieve(this_interface, timeout=timeout,\n\u001b[1;32m 513\u001b[0m \u001b[0mwhen\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwhen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeta_only\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmeta_only\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 514\u001b[0;31m version_info=version_info)\n\u001b[0m\u001b[1;32m 515\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mversion_info\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 516\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;32m~/.local/lib/python3.6/site-packages/iCalibrationDB/meta_data.py\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self, receiver, when, silent, timeout, meta_only, version_info)\u001b[0m\n\u001b[1;32m 285\u001b[0m \u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreceiver\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 286\u001b[0m \u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_pyobj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 287\u001b[0;31m \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_pyobj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 288\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 289\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0msock\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;32m/software/anaconda3/5.2/lib/python3.6/site-packages/zmq/sugar/socket.py\u001b[0m in \u001b[0;36mrecv_pyobj\u001b[0;34m(self, flags)\u001b[0m\n\u001b[1;32m 620\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0many\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mreasons\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m~\u001b[0m\u001b[0mSocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mmight\u001b[0m \u001b[0mfail\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 621\u001b[0m \"\"\"\n\u001b[0;32m--> 622\u001b[0;31m \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 623\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_deserialize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 624\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.recv\u001b[0;34m()\u001b[0m\n",
- "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.recv\u001b[0;34m()\u001b[0m\n",
- "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket._recv_copy\u001b[0;34m()\u001b[0m\n",
- "\u001b[0;32m/software/anaconda3/5.2/lib/python3.6/site-packages/zmq/backend/cython/checkrc.pxd\u001b[0m in \u001b[0;36mzmq.backend.cython.checkrc._check_rc\u001b[0;34m()\u001b[0m\n",
- "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
- ]
- }
- ],
+ "outputs": [],
"source": [
"# Retrieve list of meta-data\n",
"constant_versions = []\n",
@@ -274,11 +175,14 @@
" copy.deepcopy(mcond), None,\n",
" cal_db_interface,\n",
" creation_time=start,\n",
- " verbosity=0,\n",
+ " verbosity=2,\n",
" timeout=cal_db_timeout,\n",
" meta_only=True,\n",
" version_info=True)\n",
"\n",
+ " if not isinstance(data, list):\n",
+ " continue\n",
+ " \n",
" # Request BP constant versions\n",
" print('constantDark:', constantsDark[const], ) \n",
" dataBP = get_from_db(getattr(det, pars['module']),\n",
@@ -287,17 +191,13 @@
" copy.deepcopy(mcond), None,\n",
" cal_db_interface,\n",
" creation_time=start,\n",
- " verbosity=0,\n",
+ " verbosity=2,\n",
" timeout=cal_db_timeout,\n",
" meta_only=True,\n",
" version_info=True)\n",
" \n",
- " if not isinstance(data, list) or not isinstance(dataBP, list):\n",
- " continue\n",
- " \n",
- " found_BPmatch = False\n",
" for d in data:\n",
- " print('Item: ', d)\n",
+ " # print('Item: ', d)\n",
" # Match proper BP constant version\n",
" # and get constant version within\n",
" # requested time range\n",
@@ -313,6 +213,10 @@
" \n",
" closest_BP = None\n",
" closest_BPtime = None\n",
+ " found_BPmatch = False\n",
+ " \n",
+ " if not isinstance(dataBP, list):\n",
+ " dataBP = []\n",
" \n",
" for dBP in dataBP:\n",
" if dBP is None:\n",
@@ -349,10 +253,15 @@
" constantBP_versions.append(dBP)\n",
" constant_versions.append(d)\n",
" constant_parameters.append(copy.deepcopy(pars))\n",
- " found_BPmatch = False\n",
" break\n",
" \n",
- "print('Number of retrieved constants with a bad pixel match is {}'.format(len(constant_versions)))"
+ " if not found_BPmatch:\n",
+ " print('Bad pixels are not matched')\n",
+ " constantBP_versions.append(None)\n",
+ " constant_versions.append(d)\n",
+ " constant_parameters.append(copy.deepcopy(pars))\n",
+ " \n",
+ "print('Number of retrieved constants {}'.format(len(constant_versions)))"
]
},
{
@@ -382,6 +291,9 @@
" data = data[:, :, :, 0][..., None]\n",
" else:\n",
" data = data[..., None]\n",
+ " \n",
+ " if data.shape[2]<3:\n",
+ " data = data[:,:,0,None]\n",
"\n",
" if not isBP:\n",
" if data.shape[0] != 128:\n",
@@ -405,7 +317,7 @@
"\n",
"\n",
"ret_constants = {}\n",
- "constand_data = ConstantMetaData()\n",
+ "constant_data = ConstantMetaData()\n",
"constant_BP = ConstantMetaData()\n",
"# sort over begin_at\n",
"idxs, _ = zip(*sorted(enumerate(constant_versions), \n",
@@ -423,41 +335,46 @@
" if nconstants>0 and len(ret_constants[const][qm])>=nconstants:\n",
" continue\n",
" \n",
- " constand_data.retrieve_from_version_info(constant_versions[i])\n",
- " constant_BP.retrieve_from_version_info(constantBP_versions[i])\n",
- " \n",
- " cdata = constand_data.calibration_constant.data\n",
- " cdataBP = constant_BP.calibration_constant.data\n",
- " ctime = constand_data.calibration_constant_version.begin_at \n",
- " \n",
- " print(\"constant: {}, module {}, begin_at {}\".format(const, qm, ctime))\n",
"\n",
+ " constant_data.retrieve_from_version_info(constant_versions[i])\n",
+ " cdata = constant_data.calibration_constant.data\n",
+ " ctime = constant_data.calibration_constant_version.begin_at \n",
" cdata = modify_const(const, cdata)\n",
- " cdataBP = modify_const(const, cdataBP, True)\n",
+ " print(\"constant: {}, module {}, begin_at {}\".format(const, qm, ctime))\n",
"\n",
- " if cdataBP.shape != cdata.shape:\n",
- " print('Wrong bad pixel shape! {}, expected {}'.format(cdataBP.shape, cdata.shape))\n",
- " continue\n",
+ " if constantBP_versions[i]:\n",
+ " constant_BP.retrieve_from_version_info(constantBP_versions[i])\n",
+ " cdataBP = constant_BP.calibration_constant.data\n",
+ " cdataBP = modify_const(const, cdataBP, True)\n",
"\n",
- " # Apply bad pixel mask\n",
- " cdataABP = np.copy(cdata)\n",
- " cdataABP[cdataBP > 0] = np.nan\n",
+ " if cdataBP.shape != cdata.shape:\n",
+ " print('Wrong bad pixel shape! {}, expected {}'.format(cdataBP.shape, cdata.shape))\n",
+ " cdataBP = np.full_like(cdata, -1)\n",
"\n",
- " # Create superpixels for constants with BP applied\n",
- " cdataABP = get_rebined(cdataABP, spShape)\n",
- " toStoreBP = prepare_to_store(np.nanmean(cdataABP, axis=(1, 3)), nMem)\n",
- " toStoreBPStd = prepare_to_store(np.nanstd(cdataABP, axis=(1, 3)), nMem)\n",
+ " # Apply bad pixel mask\n",
+ " cdataABP = np.copy(cdata)\n",
+ " cdataABP[cdataBP > 0] = np.nan\n",
"\n",
- " # Prepare number of bad pixels per superpixels\n",
- " cdataBP = get_rebined(cdataBP, spShape)\n",
- " cdataNBP = prepare_to_store(np.nansum(cdataBP > 0, axis=(1, 3)), nMem)\n",
+ " # Create superpixels for constants with BP applied\n",
+ " cdataABP = get_rebined(cdataABP, spShape)\n",
+ " toStoreBP = prepare_to_store(np.nanmean(cdataABP, axis=(1, 3)), nMem)\n",
+ " toStoreBPStd = prepare_to_store(np.nanstd(cdataABP, axis=(1, 3)), nMem)\n",
+ "\n",
+ " # Prepare number of bad pixels per superpixels\n",
+ " cdataBP = get_rebined(cdataBP, spShape)\n",
+ " cdataNBP = prepare_to_store(np.nansum(cdataBP > 0, axis=(1, 3)), nMem)\n",
"\n",
" # Create superpixels for constants without BP applied\n",
" cdata = get_rebined(cdata, spShape)\n",
" toStoreStd = prepare_to_store(np.nanstd(cdata, axis=(1, 3)), nMem)\n",
" toStore = prepare_to_store(np.nanmean(cdata, axis=(1, 3)), nMem)\n",
" \n",
- " dpar = {p.name: p.value for p in constand_data.detector_condition.parameters}\n",
+ " if not constantBP_versions[i]:\n",
+ " toStoreBP = np.full_like(toStore, np.nan)\n",
+ " toStoreBPStd = np.full_like(toStore, np.nan)\n",
+ " cdataNBP = np.full_like(toStore, np.nan)\n",
+ " \n",
+ " dpar = {p.name: p.value for p in constant_data.detector_condition.parameters}\n",
"\n",
" print(\"Store values in dict\", const, qm, ctime)\n",
" ret_constants[const][qm].append({'ctime': ctime,\n",
@@ -633,7 +550,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "scrolled": true
+ "scrolled": false
},
"outputs": [],
"source": [
@@ -664,7 +581,7 @@
" cmdata = np.array(data[\"mdata\"])\n",
" for i, tick in enumerate(ctimes_ticks):\n",
" ctimes_ticks[i] = ctimes_ticks[i] + \\\n",
- " ', V={:1.0f}'.format(cmdata[i]['Sensor Bias Voltage']) + \\\n",
+ " ', A={}'.format(cmdata[i].get('Acquisition rate', None)) + \\\n",
" ', M={:1.0f}'.format(\n",
" cmdata[i]['Memory cells'])\n",
"\n",
@@ -700,12 +617,16 @@
" # Reshape: ASICs over cells for plotting\n",
" pdata = {}\n",
" for key in rdata:\n",
+ " if len(rdata[key].shape)<3:\n",
+ " continue\n",
" pdata[key] = rdata[key][:, :, :, :nMemToShow].reshape(\n",
" nTimes, nBins).swapaxes(0, 1)\n",
"\n",
" # Summary over ASICs\n",
" adata = {}\n",
" for key in rdata:\n",
+ " if len(rdata[key].shape)<3:\n",
+ " continue\n",
" adata[key] = np.nanmean(rdata[key], axis=(1, 2)).swapaxes(0, 1)\n",
"\n",
" # Plotting\n",
diff --git a/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
index e2b9a436f95556ec01a9ce894a54300cbc0bca6b..908d75feac24609e6aa28157f92efa219b076dba 100644
--- a/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
+++ b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
@@ -29,11 +29,13 @@
"bias_voltage = [79] # Bias voltage\n",
"temperature = [235, 216, 245] # Operation temperature\n",
"integration_time = [1, 50] # Integration time\n",
- "pixels_x=[1934]\n",
- "pixels_y=[960]\n",
- "max_time = 15\n",
+ "pixels_x=[1934] # number of pixels along X axis\n",
+ "pixels_y=[960] # number of pixels along Y axis\n",
+ "max_time = 15 # max time margin in minutes to match bad pixels\n",
"parameter_names = ['bias_voltage', 'integration_time', 'temperature', \n",
" 'gain_setting', 'pixels_x', 'pixels_y'] # names of parameters\n",
+ "\n",
+ "separate_plot = ['integration_time', 'gain_setting', 'temperature'] # Plot on separate plots\n",
"photon_energy = 9.2 # Photon energy of the beam\n",
"out_folder = \"/gpfs/exfel/data/scratch/karnem/test_FCCD/\" # output folder\n",
"use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
@@ -281,16 +283,10 @@
" qm = db_module\n",
" \n",
" print(\"constant: {}, module {}\".format(const,qm))\n",
- " \n",
" constant_data.retrieve_from_version_info(constant_version)\n",
" \n",
- " # Convert parameters to dict\n",
- " dpar = {p.name: p.value for p in constant_data.detector_condition.parameters}\n",
- " \n",
- " const = \"{}_{}_{}_{}\".format(const, \n",
- " constant_parameters[i]['gain_setting'],\n",
- " constant_parameters[i]['temperature'],\n",
- " constant_parameters[i]['integration_time'])\n",
+ " for key in separate_plot:\n",
+ " const = '{}_{}'.format(const, constant_parameters[i][key])\n",
" \n",
" if not const in ret_constants:\n",
" ret_constants[const] = {}\n",
@@ -405,7 +401,7 @@
"# loop over constat type\n",
"for const, modules in ret_constants.items():\n",
"\n",
- " const, gain, temp, int_time = const.split(\"_\")\n",
+ " const = const.split(\"_\")\n",
" print('Const: {}'.format(const))\n",
"\n",
" # Loop over modules\n",
@@ -437,7 +433,7 @@
" nBins = nPixels\n",
"\n",
" # Avoid too low values\n",
- " if const in [\"Noise\", \"Offset\", \"Noise-e\"]:\n",
+ " if const[0] in [\"Noise\", \"Offset\"]:\n",
" rdata['Mean'][rdata['Mean'] < 0.1] = np.nan\n",
" if 'MeanBP' in rdata:\n",
" rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
@@ -463,13 +459,16 @@
" unit = '[%]'\n",
" else:\n",
" unit = '[ADU]'\n",
- " if const == 'Noise-e':\n",
- " unit = '[$e^-$]'\n",
"\n",
- " title = '{}, module {}, gain {} {}'.format(\n",
- " const, mod, gain, keys[key][1])\n",
- " cb_label = '{}, {} {}'.format(const, keys[key][2], unit)\n",
+ " title = '{}, module {}, {}'.format(\n",
+ " const[0], mod, keys[key][1])\n",
+ " cb_label = '{}, {} {}'.format(const[0], keys[key][2], unit)\n",
"\n",
+ " fname = '{}/{}_{}'.format(out_folder, const[0], mod.replace('_', ''))\n",
+ " for item in const[1:]:\n",
+ " fname = '{}_{}'.format(fname, item)\n",
+ " fname = '{}_ASIC_{}.png'.format(fname, key)\n",
+ " \n",
" vmin,vmax = get_range(pdata[key][::-1].flatten(), plot_range)\n",
" hm_combine(pdata[key][::-1], htype=HMType.mro,\n",
" x_label='Creation Time', y_label='ASIC ID',\n",
@@ -477,8 +476,7 @@
" x_ticks=np.arange(len(ctimes_ticks))+0.3,\n",
" title=title, cb_label=cb_label,\n",
" vmin=vmin, vmax=vmax,\n",
- " fname='{}/{}_{}_g{}_t{}_t{}_ASIC_{}.png'.format(\n",
- " out_folder, const, mod.replace('_', ''), gain, temp, int_time, key),\n",
+ " fname=fname,\n",
" pad=[0.125, 0.125, 0.12, 0.185])\n"
]
}
diff --git a/notebooks/Jungfrau/PlotFromCalDB_Jungfrau_NBC.ipynb b/notebooks/Jungfrau/PlotFromCalDB_Jungfrau_NBC.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..1b428701c5d70bdf46a1e64b2a90444e0352fba2
--- /dev/null
+++ b/notebooks/Jungfrau/PlotFromCalDB_Jungfrau_NBC.ipynb
@@ -0,0 +1,574 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Statistical analysis of calibration factors#\n",
+ "\n",
+ "Author: Mikhail Karnevskiy, Steffen Hauf, Version 0.1\n",
+ "\n",
+ "Calibration constants for JungFrau detector from the data base with injection time between start_date and end_date are considered.\n",
+ "\n",
+ "To be visualized, calibration constants are averaged per group of pixels. Plots shows calibration constant over time for each constant.\n",
+ "\n",
+ "Values shown in plots are saved in h5 files."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cluster_profile = \"noDB\" # The ipcluster profile to use\n",
+ "start_date = \"2019-06-30\" # date to start investigation interval from\n",
+ "end_date = \"2019-09-01\" # date to end investigation interval at, can be \"now\"\n",
+ "dclass=\"jungfrau\" # Detector class\n",
+ "modules = [\"Jungfrau_M125\", \"Jungfrau_M260\"] # detector entry in the DB to investigate\n",
+ "constants = [\"Noise\", \"Offset\"] # constants to plot\n",
+ "nconstants = 10 # Number of time stamps to plot. If not 0, overcome start_date.\n",
+ "bias_voltage = [90, 180] # bias voltage\n",
+ "memory_cells = [1] # number of memory cells\n",
+ "pixels_x = [1024] # number of pixels along X axis\n",
+ "pixels_y = [512, 1024] # number of pixels along Y axis\n",
+ "temperature = [291] # operational temperature\n",
+ "integration_time = [50, 250] # integration time\n",
+ "gain_setting = [0] # gain stage\n",
+ "\n",
+ "parameter_names = ['bias_voltage', 'integration_time', 'pixels_x', 'pixels_y', 'gain_setting',\n",
+ " 'temperature', 'memory_cells'] # names of parameters\n",
+ "\n",
+ "separate_plot = ['integration_time'] # Plot on separate plots\n",
+ "max_time = 15 # max time margin in minutes to match bad pixels\n",
+ "photon_energy = 9.2 # Photon energy of the beam\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_JF/\" # output folder\n",
+ "use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
+ "cal_db_interface = \"tcp://max-exfl016:8016\" # the database interface to use\n",
+ "cal_db_timeout = 180000 # timeout on caldb requests\",\n",
+ "plot_range = 3 # range for plotting in units of median absolute deviations\n",
+ "spShape = [256, 64] # Shape of superpixel"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "import copy\n",
+ "import datetime\n",
+ "import dateutil.parser\n",
+ "import numpy as np\n",
+ "import os\n",
+ "import sys\n",
+ "import warnings\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "import matplotlib.pyplot as plt\n",
+ "%matplotlib inline\n",
+ "\n",
+ "from iCalibrationDB import Constants, Conditions, Detectors, ConstantMetaData\n",
+ "from cal_tools.tools import get_from_db, get_random_db_interface\n",
+ "from cal_tools.ana_tools import (save_dict_to_hdf5, load_data_from_hdf5, \n",
+ " HMType, hm_combine,\n",
+ " combine_lists, get_range)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Prepare variables\n",
+ "parameters = [globals()[x] for x in parameter_names]\n",
+ "\n",
+ "constantsDark = {'Noise': 'BadPixelsDark',\n",
+ " 'Offset': 'BadPixelsDark'}\n",
+ "print('Bad pixels data: ', constantsDark)\n",
+ "\n",
+ "# Define parameters in order to perform loop over time stamps\n",
+ "start = datetime.datetime.now() if start_date.upper() == \"NOW\" else dateutil.parser.parse(\n",
+ " start_date)\n",
+ "end = datetime.datetime.now() if end_date.upper() == \"NOW\" else dateutil.parser.parse(\n",
+ " end_date)\n",
+ "\n",
+ "# Create output folder\n",
+ "os.makedirs(out_folder, exist_ok=True)\n",
+ "\n",
+ "# Get getector conditions\n",
+ "dconstants = getattr(Constants, dclass)\n",
+ "\n",
+ "print('CalDB Interface: {}'.format(cal_db_interface))\n",
+ "print('Start time at: ', start)\n",
+ "print('End time at: ', end)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "parameter_list = combine_lists(*parameters, names = parameter_names)\n",
+ "print(parameter_list)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "# Retrieve list of meta-data\n",
+ "constant_versions = []\n",
+ "constant_parameters = []\n",
+ "constantBP_versions = []\n",
+ "\n",
+ "# Loop over constants\n",
+ "for c, const in enumerate(constants):\n",
+ " \n",
+ " for db_module in modules:\n",
+ " det = getattr(Detectors, db_module)\n",
+ " \n",
+ " if use_existing != \"\":\n",
+ " break\n",
+ "\n",
+ " # Loop over parameters\n",
+ " for pars in parameter_list:\n",
+ "\n",
+ " if (const in [\"Offset\", \"Noise\", \"SlopesPC\"] or \"DARK\" in const.upper()):\n",
+ " dcond = Conditions.Dark\n",
+ " mcond = getattr(dcond, dclass)(**pars)\n",
+ " else:\n",
+ " dcond = Conditions.Illuminated\n",
+ " mcond = getattr(dcond, dclass)(**pars,\n",
+ " photon_energy=photon_energy)\n",
+ "\n",
+ "\n",
+ "\n",
+ " print('Request: ', const, 'with paramters:', pars)\n",
+ " # Request Constant versions for given parameters and module\n",
+ " data = get_from_db(det,\n",
+ " getattr(dconstants,\n",
+ " const)(),\n",
+ " copy.deepcopy(mcond), None,\n",
+ " cal_db_interface,\n",
+ " creation_time=start,\n",
+ " verbosity=0,\n",
+ " timeout=cal_db_timeout,\n",
+ " meta_only=True,\n",
+ " version_info=True)\n",
+ "\n",
+ " if not isinstance(data, list):\n",
+ " continue\n",
+ "\n",
+ " if const in constantsDark:\n",
+ " # Request BP constant versions\n",
+ " print('constantDark:', constantsDark[const], ) \n",
+ " dataBP = get_from_db(det,\n",
+ " getattr(dconstants, \n",
+ " constantsDark[const])(),\n",
+ " copy.deepcopy(mcond), None,\n",
+ " cal_db_interface,\n",
+ " creation_time=start,\n",
+ " verbosity=0,\n",
+ " timeout=cal_db_timeout,\n",
+ " meta_only=True,\n",
+ " version_info=True)\n",
+ "\n",
+ " if not isinstance(data, list) or not isinstance(dataBP, list):\n",
+ " continue\n",
+ "\n",
+ " found_BPmatch = False\n",
+ " for d in data:\n",
+ " # Match proper BP constant version\n",
+ " # and get constant version within\n",
+ " # requested time range\n",
+ " if d is None:\n",
+ " print('Time or data is not found!')\n",
+ " continue\n",
+ "\n",
+ " dt = dateutil.parser.parse(d['begin_at'])\n",
+ "\n",
+ " if dt.replace(tzinfo=None) > end or dt.replace(tzinfo=None) < start:\n",
+ " continue\n",
+ "\n",
+ " closest_BP = None\n",
+ " closest_BPtime = None\n",
+ "\n",
+ " for dBP in dataBP:\n",
+ " if dBP is None:\n",
+ " print(\"Bad pixels are not found!\")\n",
+ " continue\n",
+ "\n",
+ " dt = dateutil.parser.parse(d['begin_at'])\n",
+ " dBPt = dateutil.parser.parse(dBP['begin_at'])\n",
+ "\n",
+ " if dt == dBPt:\n",
+ " found_BPmatch = True\n",
+ " else:\n",
+ "\n",
+ " if np.abs(dBPt-dt).seconds < (max_time*60):\n",
+ " if closest_BP is None:\n",
+ " closest_BP = dBP\n",
+ " closest_BPtime = dBPt\n",
+ " else:\n",
+ " if np.abs(dBPt-dt) < np.abs(closest_BPtime-dt):\n",
+ " closest_BP = dBP\n",
+ " closest_BPtime = dBPt\n",
+ "\n",
+ " if dataBP.index(dBP) == len(dataBP)-1:\n",
+ " if closest_BP:\n",
+ " dBP = closest_BP\n",
+ " dBPt = closest_BPtime\n",
+ " found_BPmatch = True\n",
+ " else:\n",
+ " print('Bad pixels are not found!')\n",
+ "\n",
+ " if found_BPmatch:\n",
+ " print(\"Found constant {}: begin at {}\".format(const, dt))\n",
+ " print(\"Found bad pixels at {}\".format(dBPt))\n",
+ " constantBP_versions.append(dBP)\n",
+ " constant_versions.append(d)\n",
+ " constant_parameters.append(copy.deepcopy(pars))\n",
+ " found_BPmatch = False\n",
+ " break\n",
+ " else:\n",
+ " constant_versions += data\n",
+ " constant_parameters += [copy.deepcopy(pars)]*len(data)\n",
+ "\n",
+ "# Remove dublications\n",
+ "constant_versions_tmp = []\n",
+ "constant_parameters_tmp = []\n",
+ "for i, x in enumerate(constant_versions):\n",
+ " if x not in constant_versions_tmp:\n",
+ " constant_versions_tmp.append(x)\n",
+ " constant_parameters_tmp.append(constant_parameters[i])\n",
+ " \n",
+ "constant_versions=constant_versions_tmp\n",
+ "constant_parameters=constant_parameters_tmp\n",
+ "\n",
+ "print('Number of stored constant versions is {}'.format(len(constant_versions)))\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_rebined(a, rebin):\n",
+ " return a.reshape(\n",
+ " int(a.shape[0] / rebin[0]),\n",
+ " rebin[0],\n",
+ " int(a.shape[1] / rebin[1]),\n",
+ " rebin[1],\n",
+ " a.shape[2],\n",
+ " a.shape[3])\n",
+ "\n",
+ "def modify_const(const, data, isBP = False):\n",
+ " return data\n",
+ "\n",
+ "ret_constants = {}\n",
+ "constant_data = ConstantMetaData()\n",
+ "constant_BP = ConstantMetaData()\n",
+ "\n",
+ "# sort over begin_at\n",
+ "idxs, _ = zip(*sorted(enumerate(constant_versions), \n",
+ " key=lambda x: x[1]['begin_at'], reverse=True))\n",
+ "\n",
+ "for i in idxs:\n",
+ " const = constant_versions[i]['data_set_name'].split('/')[-2]\n",
+ " qm = constant_versions[i]['physical_device']['name']\n",
+ " \n",
+ " for key in separate_plot:\n",
+ " const = '{}_{}'.format(const, constant_parameters[i][key])\n",
+ " \n",
+ " if not const in ret_constants:\n",
+ " ret_constants[const] = {}\n",
+ " if not qm in ret_constants[const]:\n",
+ " ret_constants[const][qm] = []\n",
+ " \n",
+ " if nconstants>0 and len(ret_constants[const][qm])>=nconstants:\n",
+ " continue\n",
+ " \n",
+ " print(\"constant: {}, module {}\".format(const,qm))\n",
+ " constant_data.retrieve_from_version_info(constant_versions[i])\n",
+ " \n",
+ " cdata = constant_data.calibration_constant.data\n",
+ " ctime = constant_data.calibration_constant_version.begin_at\n",
+ " cdata = modify_const(const, cdata)\n",
+ " \n",
+ " if len(constantBP_versions)>0:\n",
+ " constant_BP.retrieve_from_version_info(constantBP_versions[i])\n",
+ " cdataBP = constant_BP.calibration_constant.data\n",
+ " cdataBP = modify_const(const, cdataBP, True)\n",
+ " \n",
+ " if cdataBP.shape != cdata.shape:\n",
+ " print('Wrong bad pixel shape! {}, expected {}'.format(cdataBP.shape, cdata.shape))\n",
+ " continue\n",
+ " \n",
+ " # Apply bad pixel mask\n",
+ " cdataABP = np.copy(cdata)\n",
+ " cdataABP[cdataBP > 0] = np.nan\n",
+ " \n",
+ " # Create superpixels for constants with BP applied\n",
+ " cdataABP = get_rebined(cdataABP, spShape)\n",
+ " toStoreBP = np.nanmean(cdataABP, axis=(1, 3))\n",
+ " toStoreBPStd = np.nanstd(cdataABP, axis=(1, 3))\n",
+ "\n",
+ " # Prepare number of bad pixels per superpixels\n",
+ " cdataBP = get_rebined(cdataBP, spShape)\n",
+ " cdataNBP = np.nansum(cdataBP > 0, axis=(1, 3))\n",
+ " else:\n",
+ " toStoreBP = 0\n",
+ " toStoreBPStd = 0\n",
+ " cdataNBP = 0\n",
+ "\n",
+ " # Create superpixels for constants without BP applied\n",
+ " cdata = get_rebined(cdata, spShape)\n",
+ " toStoreStd = np.nanstd(cdata, axis=(1, 3))\n",
+ " toStore = np.nanmean(cdata, axis=(1, 3))\n",
+ " \n",
+ " # Convert parameters to dict\n",
+ " dpar = {p.name: p.value for p in constant_data.detector_condition.parameters}\n",
+ " \n",
+ " print(\"Store values in dict\", const, qm, ctime)\n",
+ " ret_constants[const][qm].append({'ctime': ctime,\n",
+ " 'nBP': cdataNBP,\n",
+ " 'dataBP': toStoreBP,\n",
+ " 'dataBPStd': toStoreBPStd,\n",
+ " 'data': toStore,\n",
+ " 'dataStd': toStoreStd,\n",
+ " 'mdata': dpar}) \n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "if use_existing == \"\":\n",
+ " print('Save data to /CalDBAna_{}_{}.h5'.format(dclass, db_module))\n",
+ " save_dict_to_hdf5(ret_constants,\n",
+ " '{}/CalDBAna_{}_{}.h5'.format(out_folder, dclass, db_module))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if use_existing == \"\":\n",
+ " fpath = '{}/CalDBAna_{}_*.h5'.format(out_folder, dclass)\n",
+ "else:\n",
+ " fpath = '{}/CalDBAna_{}_*.h5'.format(use_existing, dclass)\n",
+ "\n",
+ "print('Load data from {}'.format(fpath))\n",
+ "ret_constants = load_data_from_hdf5(fpath)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Parameters for plotting\n",
+ "\n",
+ "keys = {\n",
+ " 'Mean': ['data', '', 'Mean over pixels'],\n",
+ " 'std': ['dataStd', '', '$\\sigma$ over pixels'],\n",
+ " 'MeanBP': ['dataBP', 'Good pixels only', 'Mean over pixels'],\n",
+ " 'NBP': ['nBP', 'Fraction of BP', 'Number of BP'],\n",
+ " 'stdBP': ['dataBPStd', 'Good pixels only', '$\\sigma$ over pixels'],\n",
+ "}\n",
+ "\n",
+ "gain_name = ['High', 'Medium', 'Low']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "print('Plot calibration constants')\n",
+ "\n",
+ "# loop over constat type\n",
+ "for const, modules in ret_constants.items():\n",
+ " \n",
+ " const = const.split(\"_\")\n",
+ " for gain in range(3):\n",
+ "\n",
+ " print('Const: {}'.format(const))\n",
+ "\n",
+ " # summary over modules\n",
+ " mod_data = {}\n",
+ " mod_names = []\n",
+ " mod_times = []\n",
+ " \n",
+ " # Loop over modules\n",
+ " for mod, data in modules.items():\n",
+ " print(mod)\n",
+ "\n",
+ " ctimes = np.array(data[\"ctime\"])\n",
+ " ctimes_ticks = [x.strftime('%y-%m-%d') for x in ctimes]\n",
+ "\n",
+ " if (\"mdata\" in data):\n",
+ " cmdata = np.array(data[\"mdata\"])\n",
+ " for i, tick in enumerate(ctimes_ticks):\n",
+ " ctimes_ticks[i] = ctimes_ticks[i] + \\\n",
+ " ', V={:1.0f}'.format(cmdata[i]['Sensor Temperature']) + \\\n",
+ " ', T={:1.0f}'.format(\n",
+ " cmdata[i]['Integration Time'])\n",
+ "\n",
+ " sort_ind = np.argsort(ctimes_ticks)\n",
+ " ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
+ "\n",
+ " # Create sorted by data dataset\n",
+ " rdata = {}\n",
+ " for key, item in keys.items():\n",
+ " if item[0] in data:\n",
+ " rdata[key] = np.array(data[item[0]])[sort_ind]\n",
+ "\n",
+ " nTimes = rdata['Mean'].shape[0]\n",
+ " nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n",
+ " nBins = nPixels\n",
+ " \n",
+ " # Select gain\n",
+ " if const[0] not in [\"Gain\", \"Noise-e\"]:\n",
+ " for key in rdata:\n",
+ " if len(rdata[key].shape)<5:\n",
+ " continue\n",
+ " rdata[key] = rdata[key][..., 0, gain]\n",
+ "\n",
+ " # Avoid to low values\n",
+ " if const[0] in [\"Noise10Hz\", \"Offset10Hz\"]:\n",
+ " rdata['Mean'][rdata['Mean'] < 0.1] = np.nan\n",
+ " if 'MeanBP' in rdata:\n",
+ " rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
+ " if 'NBP' in rdata:\n",
+ " rdata['NBP'] = rdata['NBP'].astype(float)\n",
+ " rdata['NBP'][rdata['NBP'] == spShape[0]*spShape[1]] = np.nan\n",
+ "\n",
+ " # Reshape: ASICs over cells for plotting\n",
+ " pdata = {}\n",
+ " for key in rdata:\n",
+ " if len(rdata[key].shape)<3:\n",
+ " continue\n",
+ " pdata[key] = rdata[key].reshape(nTimes, nBins).swapaxes(0, 1)\n",
+ "\n",
+ " # Summary over ASICs\n",
+ " adata = {}\n",
+ " for key in rdata:\n",
+ " if len(rdata[key].shape)<3:\n",
+ " continue\n",
+ " adata[key] = np.nansum(rdata[key], axis=(1, 2))\n",
+ "\n",
+ " # Summary information over modules\n",
+ " for key in pdata:\n",
+ " if key not in mod_data:\n",
+ " mod_data[key] = []\n",
+ " if key == 'NBP':\n",
+ " mod_data[key].append(np.nansum(pdata[key], axis=0))\n",
+ " else:\n",
+ " mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
+ "\n",
+ " mod_names.append(mod)\n",
+ " mod_times.append(ctimes[sort_ind])\n",
+ " \n",
+ " # Plotting\n",
+ " for key in pdata:\n",
+ " \n",
+ " if len(pdata[key].shape)<2:\n",
+ " continue\n",
+ " \n",
+ " vmin,vmax = get_range(pdata[key][::-1].flatten(), plot_range)\n",
+ " if key == 'NBP':\n",
+ " unit = '[%]'\n",
+ " else:\n",
+ " unit = '[ADU]'\n",
+ "\n",
+ " title = '{}, module {}, {}'.format(\n",
+ " const[0], mod, keys[key][1])\n",
+ " cb_label = '{}, {} {}'.format(const[0], keys[key][2], unit)\n",
+ "\n",
+ " fname = '{}/{}_{}'.format(out_folder, const[0], mod.replace('_', ''))\n",
+ " for item in const[1:]:\n",
+ " fname = '{}_{}'.format(fname, item)\n",
+ " fname = '{}_ASIC_{}.png'.format(fname, key)\n",
+ " \n",
+ " hm_combine(pdata[key][::-1], htype=HMType.mro,\n",
+ " x_label='Creation Time', y_label='ASIC ID',\n",
+ " x_ticklabels=ctimes_ticks,\n",
+ " x_ticks=np.arange(len(ctimes_ticks))+0.3,\n",
+ " title=title, cb_label=cb_label,\n",
+ " vmin=vmin, vmax=vmax,\n",
+ " fname=fname,\n",
+ " pad=[0.125, 0.125, 0.12, 0.185])\n",
+ "\n",
+ " \n",
+ " # Summary over modules\n",
+ " for key in mod_data:\n",
+ " \n",
+ " if key == 'NBP':\n",
+ " unit = ''\n",
+ " else:\n",
+ " unit = '[ADU]'\n",
+ "\n",
+ " title = '{}, All modules, {} gain, {}'.format(\n",
+ " const[0], gain_name[gain], keys[key][1])\n",
+ " \n",
+ " fname = '{}/{}_{}'.format(out_folder, const[0], 'All')\n",
+ " for item in const[1:]:\n",
+ " fname = '{}_{}'.format(fname, item)\n",
+ " fname = '{}_ASIC_{}.png'.format(fname, key)\n",
+ " \n",
+ " fig = plt.figure(figsize=(12,12) )\n",
+ " for i in range(len(mod_data[key])):\n",
+ " plt.scatter(mod_times[i], mod_data[key][i], label=mod_names[i])\n",
+ " plt.grid()\n",
+ " plt.xlabel('Creation Time')\n",
+ " plt.ylabel('{}, {} {}'.format(const[0], keys[key][2], unit)) \n",
+ " plt.legend(loc='best guess')\n",
+ " plt.title(title)\n",
+ " fig.savefig(fname)\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb b/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
index d6eba94618588f1bc95f0dbdd81a35145501844d..00c1b59cda883ddd574b7d8484473580a275bc7d 100644
--- a/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
+++ b/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
@@ -26,13 +26,13 @@
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "start_date = \"2018-01-30\" # Date to start investigation interval from\n",
- "end_date = \"2018-12-12\" # Date to end investigation interval at, can be \"now\"\n",
+ "start_date = \"2019-01-30\" # Date to start investigation interval from\n",
+ "end_date = \"2019-12-12\" # Date to end investigation interval at, can be \"now\"\n",
"nconstants = 10 # Number of time stamps to plot. If not 0, overcome start_date.\n",
- "constants = [\"Offset\", \"Noise\", \"SlopesFF\", \"SlopesCI\"] # constants to plot\n",
+ "constants = [\"Noise\", \"Offset\", \"SlopesFF\", \"SlopesCI\"] # constants to plot\n",
"modules = [2] # Modules, set to -1 for all, range allowed\n",
- "bias_voltages = [250, 500] # Bias voltage\n",
- "mem_cells = [128, 256, 512] # Number of used memory cells.\n",
+ "bias_voltages = [250] # Bias voltage\n",
+ "mem_cells = [512] # Number of used memory cells.\n",
"photon_energy = 9.2 # Photon energy of the beam\n",
"out_folder = \"/gpfs/exfel/data/scratch/karnem/test_LPD/\" # Output folder, required\n",
"use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
@@ -49,7 +49,8 @@
"range_noise_e = [100., 600., 100., 600.] # plotting range for noise in [e-]: high gain l, r, medium gain l, r \n",
"range_slopesCI = [0.95, 1.05, 0.0, 0.5] # plotting range for slope CI: high gain l, r, medium gain l, r \n",
"range_slopesFF = [0.8, 1.2, 0.8, 1.2] # plotting range for slope FF: high gain l, r, medium gain l, r \n",
- "plot_range = 3 # range for plotting in units of median absolute deviations"
+ "plot_range = 3 # range for plotting in units of median absolute deviations\n",
+ "x_labels = ['Sensor Bias Voltage', 'Memory cells'] # parameters to be shown on X axis"
]
},
{
@@ -92,7 +93,7 @@
"modules = [\"Q{}M{}\".format(x // 4 + 1, x % 4 + 1) for x in modules]\n",
"\n",
"constantsDark = {\"SlopesFF\": 'BadPixelsFF',\n",
- " 'SlopesPC': 'BadPixelsPC',\n",
+ " 'SlopesCI': 'BadPixelsCI',\n",
" 'Noise': 'BadPixelsDark',\n",
" 'Offset': 'BadPixelsDark'}\n",
"print('Bad pixels data: ', constantsDark)\n",
@@ -146,7 +147,7 @@
" # Loop over parameters\n",
" for pars in parameter_list:\n",
" \n",
- " if (const in [\"Offset\", \"Noise\", \"SlopesPC\"] or \"DARK\" in const.upper()):\n",
+ " if (const in [\"Offset\", \"Noise\", \"SlopesCI\"] or \"DARK\" in const.upper()):\n",
" dcond = Conditions.Dark\n",
" mcond = getattr(dcond, dclass)(\n",
" memory_cells=pars['mem_cells'],\n",
@@ -171,6 +172,9 @@
" meta_only=True,\n",
" version_info=True)\n",
"\n",
+ " if not isinstance(data, list):\n",
+ " continue\n",
+ " \n",
" # Request BP constant versions\n",
" print('constantDark:', constantsDark[const], ) \n",
" dataBP = get_from_db(getattr(det, pars['module']),\n",
@@ -184,11 +188,9 @@
" meta_only=True,\n",
" version_info=True)\n",
" \n",
- " if not isinstance(data, list) or not isinstance(dataBP, list):\n",
- " continue\n",
" \n",
- " found_BPmatch = False\n",
" for d in data:\n",
+ " # print('Item: ', d)\n",
" # Match proper BP constant version\n",
" # and get constant version within\n",
" # requested time range\n",
@@ -204,6 +206,10 @@
" \n",
" closest_BP = None\n",
" closest_BPtime = None\n",
+ " found_BPmatch = False\n",
+ " \n",
+ " if not isinstance(dataBP, list):\n",
+ " dataBP = []\n",
" \n",
" for dBP in dataBP:\n",
" if dBP is None:\n",
@@ -240,10 +246,15 @@
" constantBP_versions.append(dBP)\n",
" constant_versions.append(d)\n",
" constant_parameters.append(copy.deepcopy(pars))\n",
- " found_BPmatch = False\n",
" break\n",
" \n",
- "print('Number of retrieved constants with a bad pixel match is {}'.format(len(constant_versions)))"
+ " if not found_BPmatch:\n",
+ " print('Bad pixels are not matched')\n",
+ " constantBP_versions.append(None)\n",
+ " constant_versions.append(d)\n",
+ " constant_parameters.append(copy.deepcopy(pars))\n",
+ " \n",
+ "print('Number of retrieved constants {}'.format(len(constant_versions)))"
]
},
{
@@ -287,7 +298,7 @@
"\n",
"\n",
"ret_constants = {}\n",
- "constand_data = ConstantMetaData()\n",
+ "constant_data = ConstantMetaData()\n",
"constant_BP = ConstantMetaData()\n",
"\n",
"# sort over begin_at\n",
@@ -306,41 +317,45 @@
" if nconstants>0 and len(ret_constants[const][qm])>=nconstants:\n",
" continue\n",
" \n",
- " constand_data.retrieve_from_version_info(constant_versions[i])\n",
- " constant_BP.retrieve_from_version_info(constantBP_versions[i])\n",
- " \n",
- " cdata = constand_data.calibration_constant.data\n",
- " cdataBP = constant_BP.calibration_constant.data\n",
- " ctime = constand_data.calibration_constant_version.begin_at \n",
- " \n",
+ " constant_data.retrieve_from_version_info(constant_versions[i])\n",
+ " cdata = constant_data.calibration_constant.data\n",
+ " ctime = constant_data.calibration_constant_version.begin_at \n",
+ " cdata = modify_const(const, cdata)\n",
" print(\"constant: {}, module {}, begin_at {}\".format(const, qm, ctime))\n",
"\n",
- " cdata = modify_const(const, cdata)\n",
- " cdataBP = modify_const(const, cdataBP)\n",
+ " if constantBP_versions[i]:\n",
+ " constant_BP.retrieve_from_version_info(constantBP_versions[i])\n",
+ " cdataBP = constant_BP.calibration_constant.data\n",
+ " cdataBP = modify_const(const, cdataBP)\n",
"\n",
- " if cdataBP.shape != cdata.shape:\n",
- " print('Wrong bad pixel shape! {}, expected {}'.format(cdataBP.shape, cdata.shape))\n",
- " continue\n",
+ " if cdataBP.shape != cdata.shape:\n",
+ " print('Wrong bad pixel shape! {}, expected {}'.format(cdataBP.shape, cdata.shape))\n",
+ " cdataBP = np.full_like(cdata, -1)\n",
"\n",
- " # Apply bad pixel mask\n",
- " cdataABP = np.copy(cdata)\n",
- " cdataABP[cdataBP > 0] = np.nan\n",
+ " # Apply bad pixel mask\n",
+ " cdataABP = np.copy(cdata)\n",
+ " cdataABP[cdataBP > 0] = np.nan\n",
"\n",
- " # Create superpixels for constants with BP applied\n",
- " cdataABP = get_rebined(cdataABP, spShape)\n",
- " toStoreBP = prepare_to_store(np.nanmean(cdataABP, axis=(1, 3)), nMem)\n",
- " toStoreBPStd = prepare_to_store(np.nanstd(cdataABP, axis=(1, 3)), nMem)\n",
+ " # Create superpixels for constants with BP applied\n",
+ " cdataABP = get_rebined(cdataABP, spShape)\n",
+ " toStoreBP = prepare_to_store(np.nanmean(cdataABP, axis=(1, 3)), nMem)\n",
+ " toStoreBPStd = prepare_to_store(np.nanstd(cdataABP, axis=(1, 3)), nMem)\n",
"\n",
- " # Prepare number of bad pixels per superpixels\n",
- " cdataBP = get_rebined(cdataBP, spShape)\n",
- " cdataNBP = prepare_to_store(np.nansum(cdataBP > 0, axis=(1, 3)), nMem)\n",
+ " # Prepare number of bad pixels per superpixels\n",
+ " cdataBP = get_rebined(cdataBP, spShape)\n",
+ " cdataNBP = prepare_to_store(np.nansum(cdataBP > 0, axis=(1, 3)), nMem)\n",
"\n",
" # Create superpixels for constants without BP applied\n",
" cdata = get_rebined(cdata, spShape)\n",
" toStoreStd = prepare_to_store(np.nanstd(cdata, axis=(1, 3)), nMem)\n",
" toStore = prepare_to_store(np.nanmean(cdata, axis=(1, 3)), nMem)\n",
" \n",
- " dpar = {p.name: p.value for p in constand_data.detector_condition.parameters}\n",
+ " if not constantBP_versions[i]:\n",
+ " toStoreBP = np.full_like(toStore, np.nan)\n",
+ " toStoreBPStd = np.full_like(toStore, np.nan)\n",
+ " cdataNBP = np.full_like(toStore, np.nan)\n",
+ " \n",
+ " dpar = {p.name: p.value for p in constant_data.detector_condition.parameters}\n",
"\n",
" print(\"Store values in dict\", const, qm, ctime)\n",
" ret_constants[const][qm].append({'ctime': ctime,\n",
diff --git a/notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb b/notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb
index 618f01887c9f66bc67e908131e615cde8b299c03..e1a45eb7228dd58394bcd425e0cc316b0b78129e 100644
--- a/notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb
+++ b/notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb
@@ -14,14 +14,12 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "out_folder = \"/gpfs/exfel/data/scratch/karnem/testLPD_11/\" # Output folder, required\n",
- "use_existing = \"/gpfs/exfel/data/scratch/karnem/testLPD_10/\" # Input folder\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_LPD/\" # Output folder, required\n",
+ "use_existing = \"/gpfs/exfel/data/scratch/karnem/test_LPD/\" # Input folder\n",
"dclass = \"LPD\" # Detector class\n",
"nMemToShow = 32 # Number of memory cells to be shown in plots over ASICs\n",
"range_offset = [4000., 5500, 6500, 8500] # plotting range for offset: high gain l, r, medium gain l, r \n",
@@ -30,7 +28,9 @@
"range_noise_e = [85., 500., 85., 500.] # plotting range for noise in [e-]: high gain l, r, medium gain l, r \n",
"range_slopesPC = [22.0, 27.0, -0.5, 1.5] # plotting range for slope PC: high gain l, r, medium gain l, r \n",
"range_slopesCI = [22.0, 27.0, -0.5, 1.5] # plotting range for slope CI: high gain l, r, medium gain l, r \n",
- "range_slopesFF = [0.8, 1.2, 0.6, 1.2] # plotting range for slope FF: high gain l, r, medium gain l, r "
+ "range_slopesFF = [0.8, 1.2, 0.6, 1.2] # plotting range for slope FF: high gain l, r, medium gain l, r \n",
+ "plot_range = 3 # range for plotting in units of median absolute deviations\n",
+ "x_labels = ['Acquisition rate', 'Memory cells'] # parameters to be shown on X axis"
]
},
{
@@ -48,7 +48,7 @@
"\n",
"from cal_tools.ana_tools import (load_data_from_hdf5, \n",
" HMType, multi_union,\n",
- " hm_combine)"
+ " hm_combine, get_range)"
]
},
{
@@ -81,9 +81,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Parameters for plotting\n",
@@ -116,7 +114,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "scrolled": true
+ "scrolled": false
},
"outputs": [],
"source": [
@@ -147,10 +145,9 @@
" if (\"mdata\" in data):\n",
" cmdata = np.array(data[\"mdata\"])\n",
" for i, tick in enumerate(ctimes_ticks):\n",
- " ctimes_ticks[i] = ctimes_ticks[i] + \\\n",
- " ', V={:1.0f}'.format(cmdata[i]['Sensor Bias Voltage']) + \\\n",
- " ', M={:1.0f}'.format(\n",
- " cmdata[i]['Memory cells'])\n",
+ " for entr in x_labels:\n",
+ " ctimes_ticks[i] += ', {}={}'.format(entr[0].upper(), \n",
+ " cmdata[i].get(entr, None))\n",
"\n",
" sort_ind = np.argsort(ctimes_ticks)\n",
" ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
@@ -217,15 +214,10 @@
" nModules = len(mod_names)\n",
" mod_idx = np.argsort(mod_names)\n",
" for key in mod_data:\n",
- " vmin = None\n",
- " vmax = None\n",
+ " vmin,vmax = get_range(np.array(mod_data[key])[mod_idx][::-1].flatten(), plot_range)\n",
" if const in rangevals and key in ['Mean', 'MeanBP']:\n",
" vmin = rangevals[const][gain][0]\n",
" vmax = rangevals[const][gain][1]\n",
- " else:\n",
- " vmin = np.nanmin(np.array(mod_data[key]))\n",
- " vmax = np.nanmean(\n",
- " np.array(mod_data[key])) + 2*np.nanstd(np.array(mod_data[key]))\n",
"\n",
" htype = None\n",
" if const in ['SlopesFF', 'SlopesPC', 'SlopesCI']:\n",
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index fe9df1c8a4354f8fa39da5d8d60385a94ec02c71..a004bdb0e2bc30d95687a032f0e8f6d1c9ca6c02 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -150,6 +150,13 @@ notebooks = {
"use function": "balance_sequences",
"cluster cores": 4},
},
+
+ "STATS_FROM_DB": {
+ "notebook": "notebooks/Jungfrau/PlotFromCalDB_Jungfrau_NBC.ipynb",
+ "concurrency": {"parameter": None,
+ "default concurrency": None,
+ "cluster cores": 1},
+ },
},
"EPIX": {
"DARK": {