diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5fe0821291a7b1305fa22bbd29f160a52fed9024..7ae2b28ae72dca0aff0b83c2059f13f209dde184 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -55,7 +55,6 @@ pytest:
automated_test:
variables:
OUTPUT: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
- REFERENCE: reference_folder
DETECTORS: all
CALIBRATION: all
stage: automated_test
@@ -68,7 +67,7 @@ automated_test:
- python3 -m pip install ".[automated_test]"
- echo "Running automated test. This can take sometime to finish depending on the test data."
- echo "Given variables are REFERENCE=$REFERENCE, OUTPUT=$OUTPUT, DETECTORS=$DETECTORS, CALIBRATION=$CALIBRATION"
- - python3 -m pytest ./tests/test_reference_runs --color yes --verbose --release-test --reference-folder /gpfs/exfel/data/scratch/xcaltst/test/$REFERENCE --out-folder /gpfs/exfel/data/scratch/xcaltst/test/$OUTPUT --detectors $DETECTORS --calibration $CALIBRATION
+ - python3 -m pytest ./tests/test_reference_runs --color yes --verbose --release-test --reference-folder /gpfs/exfel/d/cal_tst/reference_folder --out-folder /gpfs/exfel/data/scratch/xcaltst/test/$OUTPUT --detectors $DETECTORS --calibration $CALIBRATION
timeout: 24 hours
cython-editable-install-test:
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 88c7fac4fd4158f5807212dbe6f1a71c34b6a05e..c4431088e4a5e693f9e89cd3464c8183471240b5 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -663,8 +663,8 @@
" warning(f\"Offset constant is not available to correct {da}.\")\n",
" # Remove module from files to process.\n",
" del mapped_files[module_index_to_qm(mod)]\n",
- " karabo_da.drop(da)\n",
- " modules.drop(mod)\n",
+ " karabo_da.remove(da)\n",
+ " modules.remove(mod)\n",
"\n",
" warn_missing_constants = set(dark_constants + pc_constants + ff_constants)\n",
" warn_missing_constants -= error_missing_constants\n",
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index ba512df7877a8868ae79454b68e80a610bb9ca1c..2cbf2883903625f971734edb684e66e5c8011853 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -180,6 +180,7 @@
"else:\n",
" print(f\"Gain setting is manually set to {gain_setting}.\")\n",
"\n",
+ "force_fixed_gain_constants_flag = False\n",
"if gain_mode < 0:\n",
" gain_mode = ctrl_data.get_gain_mode()\n",
" print(f\"Gain mode is {gain_mode} ({ctrl_data.run_mode})\")\n",
@@ -188,11 +189,77 @@
" if gain_mode == 0 and memory_cells > 1:\n",
" print(\"By default fixed gain constant will be retrieved for burst mode data,\"\n",
" \" even for dynamic gain data.\")\n",
- " gain_mode = 1\n",
+ " force_fixed_gain_constants_flag = True\n",
"else:\n",
" print(f\"Gain mode is manually set to {gain_mode}.\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def jungfrau_cal_mdata(gm):\n",
+ " jf_cal = JUNGFRAU_CalibrationData(\n",
+ " detector_name=karabo_id,\n",
+ " sensor_bias_voltage=bias_voltage,\n",
+ " event_at=creation_time,\n",
+ " modules=karabo_da,\n",
+ " memory_cells=memory_cells,\n",
+ " integration_time=integration_time,\n",
+ " gain_setting=gain_setting,\n",
+ " gain_mode=gm,\n",
+ " client=rest_cfg.calibration_client(),\n",
+ " )\n",
+ "\n",
+ " constant_names = [\"Offset10Hz\", \"BadPixelsDark10Hz\"]\n",
+ " if relative_gain:\n",
+ " constant_names += [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"]\n",
+ " jf_metadata = jf_cal.metadata(calibrations=constant_names) \n",
+ " # Display retrieved calibration constants timestamps\n",
+ " jf_cal.display_markdown_retrieved_constants(metadata=jf_metadata)\n",
+ " return jf_cal, jf_metadata\n",
+ "\n",
+ "def force_fixed_gain_constants():\n",
+ " \"\"\"JF corrections in burst mode are only supported when\n",
+ " no gain switching occurs. Always retrieve fixed gain\n",
+ " constant for burst mode.\n",
+ " https://git.xfel.eu/calibration/planning/-/issues/196\n",
+ "\n",
+ " Returns:\n",
+ " dict: The metadata with the jungfrau retrieved constants.\n",
+ " {mod: {cname: ccv_metadata}}\n",
+ " \"\"\"\n",
+ " from datetime import datetime\n",
+ "\n",
+ " from cal_tools.calcat_interface import CalCatError\n",
+ "\n",
+ " try:\n",
+ " jf_cal, jf_metadata = jungfrau_cal_mdata(gm=1)\n",
+ " except CalCatError as e:\n",
+ " warning(\n",
+ " \"No fixed gain constants found. \"\n",
+ " \"Looking for dynamic gain constant. \"\n",
+ " f\"(CalCatError: {e}.\")\n",
+ "\n",
+ " jf_cal, jf_metadata = jungfrau_cal_mdata(gm=0)\n",
+ "\n",
+ " for mod, ccvs in jf_metadata.items():\n",
+ " offset = ccvs.get(\"Offset10Hz\")\n",
+ " if not offset: # This module wont be corrected later after validating constants.\n",
+ " continue\n",
+ " time_difference = creation_time - datetime.fromisoformat(offset[\"begin_validity_at\"])\n",
+ " if abs(time_difference.days) > 3:\n",
+ " warning(\n",
+ " f\"No dynamic gain constant retrieved for {mod} with at least\"\n",
+ " \" 3 days time difference with the RAW data creation date.\"\n",
+ " \" Please make sure there are available constants.\")\n",
+ " jf_metadata[mod].pop(\"Offset10Hz\")\n",
+ "\n",
+ " return jf_cal, jf_metadata"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -208,29 +275,10 @@
},
"outputs": [],
"source": [
- "jf_cal = JUNGFRAU_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " event_at=creation_time,\n",
- " modules=karabo_da,\n",
- " memory_cells=memory_cells,\n",
- " integration_time=integration_time,\n",
- " gain_setting=gain_setting,\n",
- " gain_mode=gain_mode,\n",
- " client=rest_cfg.calibration_client(),\n",
- ")\n",
- "\n",
- "da_to_pdu = {}\n",
- "for mod_info in jf_cal.physical_detector_units.values():\n",
- " da_to_pdu[mod_info[\"karabo_da\"]] = mod_info[\"physical_name\"]\n",
- "\n",
- "constant_names = [\"Offset10Hz\", \"BadPixelsDark10Hz\"]\n",
- "if relative_gain:\n",
- " constant_names += [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"]\n",
- "\n",
- "jf_metadata = jf_cal.metadata(calibrations=constant_names)\n",
- "# Display retrieved calibration constants timestamps\n",
- "jf_cal.display_markdown_retrieved_constants(metadata=jf_metadata)"
+ "if force_fixed_gain_constants_flag:\n",
+ " jf_cal, jf_metadata = force_fixed_gain_constants()\n",
+ "else:\n",
+ " jf_cal, jf_metadata = jungfrau_cal_mdata(gain_mode)"
]
},
{
@@ -274,7 +322,12 @@
"\n",
"# load constants arrays after storing fragment YAML file\n",
"# and validating constants availability.\n",
- "const_data = jf_cal.ndarray_map(metadata=jf_metadata)"
+ "const_data = jf_cal.ndarray_map(metadata=jf_metadata)\n",
+ "\n",
+ "# For plotting\n",
+ "da_to_pdu = {}\n",
+ "for mod_info in jf_cal.physical_detector_units.values():\n",
+ " da_to_pdu[mod_info[\"karabo_da\"]] = mod_info[\"physical_name\"]"
]
},
{
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index 4d8cb0e77bc15bcf1f137b7e10982bf95294371f..1b87470d3e7222a9d0dfcd5fff8ff428dc8940fc 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -12,7 +12,7 @@ import h5py
import numpy as np
import sharedmem
from dateutil import parser
-from extra_data import DataCollection, H5File, RunDirectory, by_id, components
+from extra_data import DataCollection, H5File, RunDirectory, by_id
from cal_tools import agipdalgs as calgs
from cal_tools.agipdutils import (
@@ -682,50 +682,45 @@ class AgipdCorrections:
valid_train_ids = im_dc.train_ids
# Get a count of images in each train
nimg_in_trains = im_dc[agipd_base, "image.trainId"].data_counts(False)
- nimg_in_trains = nimg_in_trains.astype(int)
+ nimg_in_trains = nimg_in_trains.astype(np.int64)
# store valid trains in shared memory
n_valid_trains = len(valid_train_ids)
data_dict["n_valid_trains"][0] = n_valid_trains
data_dict["valid_trains"][:n_valid_trains] = valid_train_ids
- if "AGIPD500K" in agipd_base:
- agipd_comp = components.AGIPD500K(im_dc)
- else:
- agipd_comp = components.AGIPD1M(im_dc)
-
- kw = {
- "unstack_pulses": False,
- }
-
# get selection for the images in this file
cm = (self.cell_sel.CM_NONE if apply_sel_pulses
else self.cell_sel.CM_PRESEL)
- cellid = np.squeeze(im_dc[agipd_base, "image.cellId"].ndarray())
+ agipd_src = im_dc[agipd_base]
+
+ cellid = agipd_src["image.cellId"].ndarray()[:, 0]
img_selected, nimg_in_trains = self.cell_sel.get_cells_on_trains(
np.array(valid_train_ids), nimg_in_trains, cellid, cm=cm)
- data_dict["nimg_in_trains"][:n_valid_trains] = nimg_in_trains
- frm_ix = np.flatnonzero(img_selected)
+ data_dict["nimg_in_trains"][:n_valid_trains] = nimg_in_trains
data_dict["cm_presel"][0] = (cm == self.cell_sel.CM_PRESEL)
- n_img = len(frm_ix)
+
+ n_img = img_selected.sum()
+ if img_selected.all():
+ # All frames selected - use slice to skip unnecessary copy
+ frm_ix = np.s_[:]
+ else:
+ frm_ix = np.flatnonzero(img_selected)
# read raw data
- # [n_modules, n_imgs, 2, x, y]
- raw_data = agipd_comp.get_array("image.data", **kw)[0]
+ # [n_imgs, 2, x, y]
+ raw_data = agipd_src['image.data'].ndarray()
# store in shmem only selected images
data_dict['nImg'][0] = n_img
data_dict['data'][:n_img] = raw_data[frm_ix, 0]
data_dict['rawgain'][:n_img] = raw_data[frm_ix, 1]
- data_dict['cellId'][:n_img] = agipd_comp.get_array(
- "image.cellId", **kw)[0, frm_ix]
- data_dict['pulseId'][:n_img] = agipd_comp.get_array(
- "image.pulseId", **kw)[0, frm_ix]
- data_dict['trainId'][:n_img] = agipd_comp.get_array(
- "image.trainId", **kw)[0, frm_ix]
+ data_dict['cellId'][:n_img] = cellid[frm_ix]
+ data_dict['pulseId'][:n_img] = agipd_src['image.pulseId'].ndarray()[frm_ix, 0]
+ data_dict['trainId'][:n_img] = agipd_src['image.trainId'].ndarray()[frm_ix, 0]
return n_img
diff --git a/tests/test_reference_runs/callab_tests.py b/tests/test_reference_runs/callab_tests.py
index 3923bf478593afcc7592cb43a75a4952b96c0a38..b566e8598b2d492f124829e5b24100cd6b7e8181 100644
--- a/tests/test_reference_runs/callab_tests.py
+++ b/tests/test_reference_runs/callab_tests.py
@@ -13,6 +13,7 @@ automated_test_config = {
"slurm-mem": "750",
"sequences": "0",
"rel-gain": True,
+ "xray-gain": True,
"n-cores-files": 2,
"ctrl-source-template": "{}/MDL/FPGA_COMP",
},
@@ -41,6 +42,7 @@ automated_test_config = {
"config": {
"blc-stripes": True,
"rel-gain": True,
+ "xray-gain": True,
"out-folder": "{}/{}/{}",
# "/gpfs/exfel/exp/SPB/202131/p900215/raw"
"in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
@@ -76,6 +78,7 @@ automated_test_config = {
"config": {
"blc-stripes": True,
"rel-gain": True,
+ "xray-gain": True,
"out-folder": "{}/{}/{}",
# "/gpfs/exfel/exp/MID/202121/p002929/raw"
"in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
@@ -95,6 +98,7 @@ automated_test_config = {
"config": {
"blc-stripes": True,
"rel-gain": True,
+ "xray-gain": True,
"out-folder": "{}/{}/{}",
# "/gpfs/exfel/exp/MID/202121/p002929/raw"
"in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
@@ -117,6 +121,7 @@ automated_test_config = {
"config": {
"blc-stripes": True,
"rel-gain": True,
+ "xray-gain": True,
"out-folder": "{}/{}/{}",
# "/gpfs/exfel/exp/MID/202121/p002929/raw"
"in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
@@ -154,6 +159,7 @@ automated_test_config = {
"config": {
"blc-stripes": True,
"rel-gain": True,
+ "xray-gain": True,
"out-folder": "{}/{}/{}",
# "/gpfs/exfel/exp/HED/202131/p900228/raw"
"in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
@@ -320,10 +326,11 @@ automated_test_config = {
"cal_type": "DARK",
"config": {
"out-folder": "{}/{}/{}",
+ # /gpfs/exfel/exp/SPB/202130/p900204/raw
"in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
- "run-high": "9004",
- "run-med": "9005",
- "run-low": "9006",
+ "run-high": "9004", # 88
+ "run-med": "9005", # 89
+ "run-low": "9006", # 90
"karabo-id-control": "",
"karabo-id": "SPB_IRDA_JF4M",
"karabo-da": [
@@ -512,8 +519,9 @@ automated_test_config = {
"cal_type": "DARK",
"config": {
"out-folder": "{}/{}/{}",
+ # Original proposal: 2655
"in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
- "run": "9045", # Original run: 224 proposal 2655
+ "run": "9045", # Original run: 224
"karabo-id": "MID_EXP_EPIX-2",
"karabo-da": "EPIX02",
},
@@ -602,6 +610,37 @@ automated_test_config = {
},
"reference-folder": "{}/{}/{}",
},
+ "SQS_DET_DSSC1M-1-DARK": {
+ "det_type": "DSSC",
+ "cal_type": "DARK",
+ "config": {
+ "out-folder": "{}/{}/{}",
+ # "/gpfs/exfel/exp/SQS/202131/p900210/raw"
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ "run": "9029", # Original run: "437",
+ "karabo-id": "SQS_DET_DSSC1M-1",
+ "slow-data-path": "SQS_NQS_DSSC/FPGA/PPT_Q",
+ "slow-data-aggregators":
+ - 1
+ - 2
+ - 3
+ - 4
+ },
+ "reference-folder": "{}/{}/{}",
+ },
+ "SQS_DET_DSSC1M-1-CORRECT": {
+ "det_type": "DSSC",
+ "cal_type": "CORRECT",
+ "config": {
+ "out-folder": "{}/{}/{}",
+ # "/gpfs/exfel/exp/SQS/202131/p900210/raw"
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ "run": "9029", # Original run: "437",
+ "karabo-id": "SQS_DET_DSSC1M-1",
+ "slow-data-path": "SQS_NQS_DSSC/FPGA/PPT_Q",
+ },
+ "reference-folder": "{}/{}/{}",
+ },
# "FXE_XAD_G2XES-DARK": {
# "det_type": "Gotthard2",
# "cal_type": "DARK",