diff --git a/.gitignore b/.gitignore
index 3ea4aabcd93099c2a4b839a5193e261e1d62578c..478939103a8e7b13d12cb825d6eaf90ef968a770 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,7 @@
LPD/results
Test
build
+coverage.*
docs/build
docs/source/_notebooks
docs/source/_static/reports
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2b56d9b29aa23836100568a593c222170bd2b096..0b44090166b6bdd18b893b8a4fd194e945a83505 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -43,7 +43,12 @@ pytest:
<<: *before_script
script:
- python3 -m pip install ".[test]"
- - python3 -m pytest --verbose --cov=cal_tools --cov=xfel_calibrate
+ - python3 -m pytest --color yes --verbose --cov=cal_tools --cov=xfel_calibrate
+# Nope... https://docs.gitlab.com/12.10/ee/user/project/merge_requests/test_coverage_visualization.html#enabling-the-feature
+# - coverage xml
+# artifacts:
+# reports:
+# cobertura: coverage.xml
cython-editable-install-test:
stage: test
@@ -51,4 +56,4 @@ cython-editable-install-test:
<<: *before_script
script:
- python3 -m pip install -e ".[test]"
- - python3 -m pytest ./tests/test_agipdalgs.py
+ - python3 -m pytest --color yes --verbose ./tests/test_agipdalgs.py
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 722eb0a113265f456e3044a81ff7ff7dd31c30d3..8aee1ac01698ad49dfa214d937779f92acb06c16 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -86,6 +86,7 @@
"import h5py\n",
"import matplotlib\n",
"import numpy as np\n",
+ "import pasha as psh\n",
"import tabulate\n",
"import yaml\n",
"\n",
@@ -95,6 +96,7 @@
"\n",
"%matplotlib inline\n",
"\n",
+ "import itertools\n",
"import multiprocessing\n",
"\n",
"from cal_tools.agipdlib import (\n",
@@ -104,9 +106,7 @@
" get_gain_setting,\n",
" get_num_cells,\n",
")\n",
- "\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
- "\n",
"from cal_tools.plotting import (\n",
" create_constant_overview,\n",
" plot_badpix_3d,\n",
@@ -125,7 +125,7 @@
" save_const_to_h5,\n",
" send_to_db,\n",
")\n",
- "from iCalibrationDB import Conditions, Constants, Detectors"
+ "import iCalibrationDB"
]
},
{
@@ -211,7 +211,7 @@
" gsettings.append(get_gain_setting(control_fname, h5path_ctrl))\n",
" if not all(g == gsettings[0] for g in gsettings):\n",
" raise ValueError(f\"Different gain settings for the 3 input runs {gsettings}\")\n",
- " gain_setting = gsettings[0]\n",
+ " gain_setting = gsettings[0]\n",
" except Exception as e:\n",
" print(f'Error while reading gain setting from: \\n{control_fname}')\n",
" print(f'Error: {e}')\n",
@@ -245,7 +245,7 @@
"print(\"Parameters are:\")\n",
"print(f\"Proposal: {prop}\")\n",
"print(f\"Memory cells: {mem_cells}/{max_cells}\")\n",
- "print(\"Runs: {}\".format([ v for v in offset_runs.values()]))\n",
+ "print(\"Runs: {}\".format([v for v in offset_runs.values()]))\n",
"print(f\"Sequences: {sequences}\")\n",
"print(f\"Interlaced mode: {interlaced}\")\n",
"print(f\"Using DB: {db_output}\")\n",
@@ -256,35 +256,6 @@
"print(f\"Operation mode is {'fixed' if fixed_gain_mode else 'adaptive'} gain mode\")"
]
},
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "The following lines will create a queue of files which will the be executed module-parallel. Distiguishing between different gains."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# set everything up filewise\n",
- "os.makedirs(out_folder, exist_ok=True)\n",
- "gmf = map_gain_stages(in_folder, offset_runs, path_template, karabo_da, sequences)\n",
- "gain_mapped_files, total_sequences, total_file_size = gmf\n",
- "print(f\"Will process a total of {total_sequences} files.\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Calculate Offsets, Noise and Thresholds ##\n",
- "\n",
- "The calculation is performed per-pixel and per-memory-cell. Offsets are simply the median value for a set of dark data taken at a given gain, noise the standard deviation, and gain-bit values the medians of the gain array."
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -306,7 +277,7 @@
" thresholds_offset_hard_mg,\n",
" thresholds_offset_hard_lg,\n",
" ]\n",
- "print(f\"Will use the following hard offset thresholds\")\n",
+ "print(\"Will use the following hard offset thresholds\")\n",
"for name, value in zip((\"High\", \"Medium\", \"Low\"), thresholds_offset_hard):\n",
" print(f\"- {name} gain: {value}\")\n",
"\n",
@@ -320,13 +291,64 @@
" ]"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The following lines will create a queue of files which will the be executed module-parallel. Distiguishing between different gains."
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
- "def characterize_module(fast_data_filename: str, channel: int, gg: int) -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
+ "# set everything up filewise\n",
+ "os.makedirs(out_folder, exist_ok=True)\n",
+ "gain_mapped_files, total_files, total_file_size = map_gain_stages(\n",
+ " in_folder, offset_runs, path_template, karabo_da, sequences\n",
+ ")\n",
+ "print(f\"Will process a total of {total_files} files ({total_file_size:.02f} GB).\")\n",
+ "\n",
+ "inp = []\n",
+ "for gain_index, (gain, qm_file_map) in enumerate(gain_mapped_files.items()):\n",
+ " for module_index in modules:\n",
+ " qm = module_index_to_qm(module_index)\n",
+ " if qm not in qm_file_map:\n",
+ " print(f\"Did not find files for {qm}\")\n",
+ " continue\n",
+ " file_queue = qm_file_map[qm]\n",
+ " while not file_queue.empty():\n",
+ " filename = file_queue.get()\n",
+ " print(f\"Process {filename} for {qm}\")\n",
+ " inp.append((filename, module_index, gain_index))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Calculate Offsets, Noise and Thresholds ##\n",
+ "\n",
+ "The calculation is performed per-pixel and per-memory-cell. Offsets are simply the median value for a set of dark data taken at a given gain, noise the standard deviation, and gain-bit values the medians of the gain array."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# min() only relevant if running on multiple modules (i.e. within notebook)\n",
+ "parallel_num_procs = min(12, total_files)\n",
+ "parallel_num_threads = multiprocessing.cpu_count() // parallel_num_procs\n",
+ "print(f\"Will use {parallel_num_procs} processes with {parallel_num_threads} threads each\")\n",
+ "\n",
+ "\n",
+ "def characterize_module(\n",
+ " fast_data_filename: str, channel: int, gain_index: int\n",
+ ") -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
" if max_cells == 0:\n",
" num_cells = get_num_cells(fast_data_filename, karabo_id, channel)\n",
" else:\n",
@@ -335,14 +357,14 @@
" print(f\"Using {num_cells} memory cells\")\n",
"\n",
" if acq_rate == 0.:\n",
- " slow_paths = control_names[gg], karabo_id_control\n",
+ " slow_paths = control_names[gain_index], karabo_id_control\n",
" fast_paths = fast_data_filename, karabo_id, channel\n",
" local_acq_rate = get_acq_rate(fast_paths, slow_paths)\n",
" else:\n",
" local_acq_rate = acq_rate\n",
"\n",
- " local_thresholds_offset_hard = thresholds_offset_hard[gg]\n",
- " local_thresholds_noise_hard = thresholds_noise_hard[gg]\n",
+ " local_thresholds_offset_hard = thresholds_offset_hard[gain_index]\n",
+ " local_thresholds_noise_hard = thresholds_noise_hard[gain_index]\n",
"\n",
" h5path_f = h5path.format(channel)\n",
" h5path_idx_f = h5path_idx.format(channel)\n",
@@ -362,64 +384,76 @@
" last_index = int(last[status != 0][-1]) + 1\n",
" first_index = int(first[status != 0][0])\n",
" im = np.array(infile[f\"{h5path_f}/data\"][first_index:last_index,...])\n",
- " cellIds = np.squeeze(infile[f\"{h5path_f}/cellId\"][first_index:last_index,...])\n",
- " \n",
+ " cell_ids = np.squeeze(infile[f\"{h5path_f}/cellId\"][first_index:last_index,...])\n",
+ "\n",
" if interlaced:\n",
" if not fixed_gain_mode:\n",
" ga = im[1::2, 0, ...]\n",
" im = im[0::2, 0, ...].astype(np.float32)\n",
- " cellIds = cellIds[::2]\n",
+ " cell_ids = cell_ids[::2]\n",
" else:\n",
" if not fixed_gain_mode:\n",
" ga = im[:, 1, ...]\n",
" im = im[:, 0, ...].astype(np.float32)\n",
"\n",
- " im = np.rollaxis(im, 2)\n",
- " im = np.rollaxis(im, 2, 1)\n",
- "\n",
+ " im = np.transpose(im)\n",
" if not fixed_gain_mode:\n",
- " ga = np.rollaxis(ga, 2)\n",
- " ga = np.rollaxis(ga, 2, 1)\n",
- " \n",
- " offset = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
- " noise = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ " ga = np.transpose(ga)\n",
+ "\n",
+ " context = psh.context.ThreadContext(num_workers=parallel_num_threads)\n",
+ " offset = context.alloc(shape=(im.shape[0], im.shape[1], num_cells), dtype=np.float64)\n",
+ " noise = context.alloc(like=offset)\n",
"\n",
" if fixed_gain_mode:\n",
" gains = None\n",
" gains_std = None\n",
" else:\n",
- " gains = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
- " gains_std = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
- "\n",
- " for cc in np.unique(cellIds[cellIds < num_cells]):\n",
- " cellidx = cellIds == cc\n",
- " offset[...,cc] = np.median(im[..., cellidx], axis=2)\n",
- " noise[...,cc] = np.std(im[..., cellidx], axis=2)\n",
+ " gains = context.alloc(like=offset)\n",
+ " gains_std = context.alloc(like=offset)\n",
+ "\n",
+ " def process_cell(worker_id, array_index, cell_number):\n",
+ " cell_slice_index = (cell_ids == cell_number)\n",
+ " im_slice = im[..., cell_slice_index]\n",
+ " offset[..., cell_number] = np.median(im_slice, axis=2)\n",
+ " noise[..., cell_number] = np.std(im_slice, axis=2)\n",
" if not fixed_gain_mode:\n",
- " gains[...,cc] = np.median(ga[..., cellidx], axis=2)\n",
- " gains_std[...,cc] = np.std(ga[..., cellidx], axis=2)\n",
+ " ga_slice = ga[..., cell_slice_index]\n",
+ " gains[..., cell_number] = np.median(ga_slice, axis=2)\n",
+ " gains_std[..., cell_number] = np.std(ga_slice, axis=2)\n",
+ "\n",
+ " context.map(process_cell, np.unique(cell_ids))\n",
"\n",
" # bad pixels\n",
- " bp = np.zeros(offset.shape, np.uint32)\n",
+ " bp = np.zeros_like(offset, dtype=np.uint32)\n",
" # offset related bad pixels\n",
" offset_mn = np.nanmedian(offset, axis=(0,1))\n",
" offset_std = np.nanstd(offset, axis=(0,1))\n",
"\n",
" bp[(offset < offset_mn-thresholds_offset_sigma*offset_std) |\n",
- " (offset > offset_mn+thresholds_offset_sigma*offset_std)] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
- " bp[(offset < local_thresholds_offset_hard[0]) | (\n",
- " offset > local_thresholds_offset_hard[1])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
- " bp[~np.isfinite(offset)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
+ " (offset > offset_mn+thresholds_offset_sigma*offset_std)] |= BadPixels.OFFSET_OUT_OF_THRESHOLD\n",
+ " bp[(offset < local_thresholds_offset_hard[0]) |\n",
+ " (offset > local_thresholds_offset_hard[1])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD\n",
+ " bp[~np.isfinite(offset)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR\n",
"\n",
" # noise related bad pixels\n",
" noise_mn = np.nanmedian(noise, axis=(0,1))\n",
" noise_std = np.nanstd(noise, axis=(0,1))\n",
" bp[(noise < noise_mn-thresholds_noise_sigma*noise_std) |\n",
- " (noise > noise_mn+thresholds_noise_sigma*noise_std)] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
- " bp[(noise < local_thresholds_noise_hard[0]) | (noise > local_thresholds_noise_hard[1])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
- " bp[~np.isfinite(noise)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
+ " (noise > noise_mn+thresholds_noise_sigma*noise_std)] |= BadPixels.NOISE_OUT_OF_THRESHOLD\n",
+ " bp[(noise < local_thresholds_noise_hard[0]) | (noise > local_thresholds_noise_hard[1])] |= BadPixels.NOISE_OUT_OF_THRESHOLD\n",
+ " bp[~np.isfinite(noise)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR\n",
"\n",
- " return offset, noise, gains, gains_std, gg, bp, num_cells, local_acq_rate"
+ " return offset, noise, gains, gains_std, bp, num_cells, local_acq_rate"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with multiprocessing.Pool(processes=parallel_num_procs) as pool:\n",
+ " results = pool.starmap(characterize_module, inp)"
]
},
{
@@ -434,49 +468,30 @@
"if not fixed_gain_mode:\n",
" gain_g = OrderedDict()\n",
" gainstd_g = OrderedDict()\n",
- " \n",
- "start = datetime.now()\n",
+ "\n",
"all_cells = []\n",
"all_acq_rate = []\n",
"\n",
- "inp = []\n",
- "for gg, (gain, mapped_files) in enumerate(gain_mapped_files.items()):\n",
- " dones = []\n",
- " for i in modules:\n",
- " qm = module_index_to_qm(i)\n",
- " if qm in mapped_files and not mapped_files[qm].empty():\n",
- " fname_in = mapped_files[qm].get()\n",
- " print(\"Process file: \", fname_in)\n",
- " dones.append(mapped_files[qm].empty())\n",
- " else:\n",
- " continue\n",
- " inp.append((fname_in, i, gg))\n",
- "\n",
- "with multiprocessing.Pool() as pool:\n",
- " results = pool.starmap(characterize_module, inp)\n",
- "\n",
- "for offset, noise, gains, gains_std, gg, bp, thiscell, thisacq in results:\n",
+ "for (_, module_index, gain_index), (offset, noise, gains, gains_std, bp,\n",
+ " thiscell, thisacq) in zip(inp, results):\n",
" all_cells.append(thiscell)\n",
" all_acq_rate.append(thisacq)\n",
- " for i in modules:\n",
- " qm = module_index_to_qm(i)\n",
- " if qm not in offset_g:\n",
- " offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
- " noise_g[qm] = np.zeros_like(offset_g[qm])\n",
- " badpix_g[qm] = np.zeros_like(offset_g[qm], np.uint32)\n",
- " if not fixed_gain_mode:\n",
- " gain_g[qm] = np.zeros_like(offset_g[qm])\n",
- " gainstd_g[qm] = np.zeros_like(offset_g[qm])\n",
- "\n",
- " offset_g[qm][...,gg] = offset\n",
- " noise_g[qm][...,gg] = noise\n",
- " badpix_g[qm][...,gg] = bp\n",
+ " qm = module_index_to_qm(module_index)\n",
+ " if qm not in offset_g:\n",
+ " offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
+ " noise_g[qm] = np.zeros_like(offset_g[qm])\n",
+ " badpix_g[qm] = np.zeros_like(offset_g[qm], np.uint32)\n",
" if not fixed_gain_mode:\n",
- " gain_g[qm][...,gg] = gains\n",
- " gainstd_g[qm][..., gg] = gains_std\n",
+ " gain_g[qm] = np.zeros_like(offset_g[qm])\n",
+ " gainstd_g[qm] = np.zeros_like(offset_g[qm])\n",
"\n",
+ " offset_g[qm][..., gain_index] = offset\n",
+ " noise_g[qm][..., gain_index] = noise\n",
+ " badpix_g[qm][..., gain_index] = bp\n",
+ " if not fixed_gain_mode:\n",
+ " gain_g[qm][..., gain_index] = gains\n",
+ " gainstd_g[qm][..., gain_index] = gains_std\n",
"\n",
- "duration = (datetime.now() - start).total_seconds()\n",
"\n",
"max_cells = np.max(all_cells)\n",
"print(f\"Using {max_cells} memory cells\")\n",
@@ -490,13 +505,16 @@
"metadata": {},
"outputs": [],
"source": [
- "# Add a badpixel due to bad gain separation\n",
+ "# Add bad pixels due to bad gain separation\n",
"if not fixed_gain_mode:\n",
- " for g in range(2):\n",
- " # Bad pixels during bad gain separation.\n",
- " # Fraction of pixels in the module with separation lower than \"thresholds_gain_sigma\".\n",
- " bad_sep = (gain_g[qm][..., g+1] - gain_g[qm][..., g]) / np.sqrt(gainstd_g[qm][..., g+1]**2 + gainstd_g[qm][..., g]**2)\n",
- " badpix_g[qm][...,g+1][(bad_sep)<thresholds_gain_sigma]|= BadPixels.GAIN_THRESHOLDING_ERROR.value"
+ " for qm in gain_g.keys():\n",
+ " for g in range(2):\n",
+ " # Bad pixels during bad gain separation.\n",
+ " # Fraction of pixels in the module with separation lower than \"thresholds_gain_sigma\".\n",
+ " bad_sep = (gain_g[qm][..., g+1] - gain_g[qm][..., g]) / \\\n",
+ " np.sqrt(gainstd_g[qm][..., g+1]**2 + gainstd_g[qm][..., g]**2)\n",
+ " badpix_g[qm][...,g+1][bad_sep<thresholds_gain_sigma] |= \\\n",
+ " BadPixels.GAIN_THRESHOLDING_ERROR"
]
},
{
@@ -553,23 +571,6 @@
"report = get_report(out_folder)"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# TODO: add db_module when received from myMDC\n",
- "# Create the modules dict of karabo_das and PDUs\n",
- "qm_dict = OrderedDict()\n",
- "for i, k_da in zip(modules, karabo_da):\n",
- " qm = module_index_to_qm(i)\n",
- " qm_dict[qm] = {\n",
- " \"karabo_da\": k_da,\n",
- " \"db_module\": \"\"\n",
- " }"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -578,11 +579,13 @@
"source": [
"# set the operating condition\n",
"# note: iCalibrationDB only adds gain_mode if it is truthy, so we don't need to handle None\n",
- "condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
- " bias_voltage=bias_voltage,\n",
- " acquisition_rate=acq_rate,\n",
- " gain_setting=gain_setting,\n",
- " gain_mode=fixed_gain_mode)"
+ "condition = iCalibrationDB.Conditions.Dark.AGIPD(\n",
+ " memory_cells=max_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " acquisition_rate=acq_rate,\n",
+ " gain_setting=gain_setting,\n",
+ " gain_mode=fixed_gain_mode\n",
+ ")"
]
},
{
@@ -591,45 +594,26 @@
"metadata": {},
"outputs": [],
"source": [
- "# Retrieve existing constants for comparison\n",
- "old_const = {}\n",
- "old_mdata = {}\n",
- "\n",
- "print('Retrieve pre-existing constants for comparison.')\n",
- "for qm in res:\n",
- " qm_db = qm_dict[qm]\n",
- " karabo_da = qm_db[\"karabo_da\"]\n",
- " for const in res[qm]:\n",
- " dconst = getattr(Constants.AGIPD, const)()\n",
- "\n",
- " # This should be used in case of running notebook\n",
- " # by a different method other than myMDC which already\n",
- " # sends CalCat info.\n",
- " # TODO: Set db_module to \"\" by default in the first cell\n",
- " if not qm_db[\"db_module\"]:\n",
- " qm_db[\"db_module\"] = get_pdu_from_db(karabo_id, karabo_da, dconst,\n",
- " condition, cal_db_interface,\n",
- " snapshot_at=creation_time)[0]\n",
- "\n",
- " data, mdata = get_from_db(karabo_id, karabo_da,\n",
- " dconst, condition,\n",
- " None, cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " verbosity=2, timeout=cal_db_timeout)\n",
- "\n",
- " old_const[const] = data\n",
- " if mdata is not None and data is not None:\n",
- " time = mdata.calibration_constant_version.begin_at\n",
- " old_mdata[const] = time.isoformat()\n",
- " os.makedirs('{}/old/'.format(out_folder), exist_ok=True)\n",
- " save_const_to_h5(qm_db[\"db_module\"], karabo_id,\n",
- " dconst, condition, data,\n",
- " file_loc, report, creation_time,\n",
- " f'{out_folder}/old/')\n",
- " else:\n",
- " old_mdata[const] = \"Not found\"\n",
- " with open(f\"{out_folder}/module_mapping_{qm}.yml\",\"w\") as fd:\n",
- " yaml.safe_dump({\"module_mapping\": {qm: qm_db[\"db_module\"]}}, fd)"
+ "# Create mapping from module(s) (qm) to karabo_da(s) and PDU(s)\n",
+ "qm_dict = OrderedDict()\n",
+ "all_pdus = get_pdu_from_db(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " constant=iCalibrationDB.CalibrationConstant(),\n",
+ " condition=condition,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " snapshot_at=creation_time.isoformat(),\n",
+ " timeout=cal_db_timeout\n",
+ ")\n",
+ "for module_index, module_da, module_pdu in zip(modules, karabo_da, all_pdus):\n",
+ " qm = module_index_to_qm(module_index)\n",
+ " qm_dict[qm] = {\n",
+ " \"karabo_da\": module_da,\n",
+ " \"db_module\": module_pdu\n",
+ " }\n",
+ " # saving mapping information for summary notebook\n",
+ " with open(f\"{out_folder}/module_mapping_{qm}.yml\", \"w\") as fd:\n",
+ " yaml.safe_dump({\"module_mapping\": {qm: module_pdu}}, fd)"
]
},
{
@@ -641,10 +625,9 @@
"md = None\n",
"\n",
"for qm in res:\n",
- " karabo_da = qm_dict[qm][\"karabo_da\"]\n",
" db_module = qm_dict[qm][\"db_module\"]\n",
" for const in res[qm]:\n",
- " dconst = getattr(Constants.AGIPD, const)()\n",
+ " dconst = getattr(iCalibrationDB.Constants.AGIPD, const)()\n",
" dconst.data = res[qm][const]\n",
"\n",
" if db_output:\n",
@@ -655,7 +638,7 @@
" if local_output:\n",
" md = save_const_to_h5(db_module, karabo_id, dconst, condition, dconst.data,\n",
" file_loc, report, creation_time, out_folder)\n",
- " print(f\"Calibration constant {const} is stored locally.\\n\")\n",
+ " print(f\"Calibration constant {const} for {qm} is stored locally in {file_loc}.\\n\")\n",
"\n",
" print(\"Constants parameter conditions are:\\n\")\n",
" print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
@@ -664,6 +647,50 @@
" f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Start retrieving existing constants for comparison\n",
+ "qm_x_const = [(qm, const) for const in res[qm] for qm in res]\n",
+ "\n",
+ "\n",
+ "def retrieve_old_constant(qm, const):\n",
+ " dconst = getattr(iCalibrationDB.Constants.AGIPD, const)()\n",
+ "\n",
+ " # This should be used in case of running notebook\n",
+ " # by a different method other than myMDC which already\n",
+ " # sends CalCat info.\n",
+ " # TODO: Set db_module to \"\" by default in the first cell\n",
+ "\n",
+ " data, mdata = get_from_db(\n",
+ " karabo_id,\n",
+ " qm_dict[qm][\"karabo_da\"],\n",
+ " constant=dconst,\n",
+ " condition=condition,\n",
+ " empty_constant=None,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " verbosity=2,\n",
+ " timeout=cal_db_timeout\n",
+ " )\n",
+ "\n",
+ " if mdata is None or data is None:\n",
+ " timestamp = \"Not found\"\n",
+ " else:\n",
+ " timestamp = mdata.calibration_constant_version.begin_at.isoformat()\n",
+ " \n",
+ " return data, timestamp\n",
+ "\n",
+ "old_retrieval_pool = multiprocessing.Pool()\n",
+ "old_retrieval_res = old_retrieval_pool.starmap_async(\n",
+ " retrieve_old_constant, qm_x_const\n",
+ ")\n",
+ "old_retrieval_pool.close()"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -674,7 +701,7 @@
"for i in modules:\n",
" qm = module_index_to_qm(i)\n",
" mnames.append(qm)\n",
- " display(Markdown(f'## Position of the module {qm} and its ASICs##'))\n",
+ " display(Markdown(f'## Position of the module {qm} and its ASICs'))\n",
"show_processed_modules(dinstance, constants=None, mnames=mnames, mode=\"position\")"
]
},
@@ -747,15 +774,17 @@
"metadata": {},
"outputs": [],
"source": [
- "cols = {BadPixels.NOISE_OUT_OF_THRESHOLD.value: (BadPixels.NOISE_OUT_OF_THRESHOLD.name, '#FF000080'),\n",
- " BadPixels.OFFSET_NOISE_EVAL_ERROR.value: (BadPixels.OFFSET_NOISE_EVAL_ERROR.name, '#0000FF80'),\n",
- " BadPixels.OFFSET_OUT_OF_THRESHOLD.value: (BadPixels.OFFSET_OUT_OF_THRESHOLD.name, '#00FF0080'),\n",
- " BadPixels.GAIN_THRESHOLDING_ERROR.value: (BadPixels.GAIN_THRESHOLDING_ERROR.name, '#FF40FF40'),\n",
- " BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value: ('OFFSET_OUT_OF_THRESHOLD + NOISE_OUT_OF_THRESHOLD', '#DD00DD80'),\n",
- " BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value |\n",
- " BadPixels.GAIN_THRESHOLDING_ERROR.value: ('MIXED', '#BFDF009F')}\n",
- "\n",
"if high_res_badpix_3d:\n",
+ " cols = {\n",
+ " BadPixels.NOISE_OUT_OF_THRESHOLD: (BadPixels.NOISE_OUT_OF_THRESHOLD.name, '#FF000080'),\n",
+ " BadPixels.OFFSET_NOISE_EVAL_ERROR: (BadPixels.OFFSET_NOISE_EVAL_ERROR.name, '#0000FF80'),\n",
+ " BadPixels.OFFSET_OUT_OF_THRESHOLD: (BadPixels.OFFSET_OUT_OF_THRESHOLD.name, '#00FF0080'),\n",
+ " BadPixels.GAIN_THRESHOLDING_ERROR: (BadPixels.GAIN_THRESHOLDING_ERROR.name, '#FF40FF40'),\n",
+ " BadPixels.OFFSET_OUT_OF_THRESHOLD | BadPixels.NOISE_OUT_OF_THRESHOLD: ('OFFSET_OUT_OF_THRESHOLD + NOISE_OUT_OF_THRESHOLD', '#DD00DD80'),\n",
+ " BadPixels.OFFSET_OUT_OF_THRESHOLD | BadPixels.NOISE_OUT_OF_THRESHOLD |\n",
+ " BadPixels.GAIN_THRESHOLDING_ERROR: ('MIXED', '#BFDF009F')\n",
+ " }\n",
+ "\n",
" display(Markdown(\"\"\"\n",
"\n",
" ## Global Bad Pixel Behaviour ##\n",
@@ -819,7 +848,6 @@
" bp_thresh[mod][...,:2] = con[...,:2]\n",
" bp_thresh[mod][...,2:] = con\n",
"\n",
- "\n",
" create_constant_overview(thresholds_g, \"Threshold (ADU)\", max_cells, 4000, 10000, 5,\n",
" badpixels=[bp_thresh, np.nan],\n",
" gmap=['HG-MG Threshold', 'MG-LG Threshold', 'High gain', 'Medium gain', 'low gain'],\n",
@@ -854,9 +882,27 @@
"metadata": {},
"outputs": [],
"source": [
- "display(Markdown('The following pre-existing constants are used for comparison: \\n'))\n",
- "for key in old_mdata:\n",
- " display(Markdown('**{}** at {}'.format(key, old_mdata[key])))"
+ "# now we need the old constants\n",
+ "old_const = {}\n",
+ "old_mdata = {}\n",
+ "old_retrieval_res.wait()\n",
+ "\n",
+ "for (qm, const), (data, timestamp) in zip(qm_x_const, old_retrieval_res.get()):\n",
+ " old_const.setdefault(qm, {})[const] = data\n",
+ " old_mdata.setdefault(qm, {})[const] = timestamp"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown(\"The following pre-existing constants are used for comparison:\"))\n",
+ "for qm, consts in old_mdata.items():\n",
+ " display(Markdown(f\"- {qm}\"))\n",
+ " for const in consts:\n",
+ " display(Markdown(f\" - {const} at {consts[const]}\"))"
]
},
{
@@ -870,7 +916,6 @@
"bits = [BadPixels.NOISE_OUT_OF_THRESHOLD, BadPixels.OFFSET_OUT_OF_THRESHOLD, BadPixels.OFFSET_NOISE_EVAL_ERROR, BadPixels.GAIN_THRESHOLDING_ERROR]\n",
"for qm in badpix_g.keys():\n",
" for gain in range(3):\n",
- "\n",
" l_data = []\n",
" l_data_old = []\n",
"\n",
@@ -878,14 +923,14 @@
" datau32 = data.astype(np.uint32)\n",
" l_data.append(len(datau32[datau32>0].flatten()))\n",
" for bit in bits:\n",
- " l_data.append(np.count_nonzero(badpix_g[qm][:,:,:,gain] & bit.value))\n",
+ " l_data.append(np.count_nonzero(badpix_g[qm][:,:,:,gain] & bit))\n",
"\n",
- " if old_const['BadPixelsDark'] is not None:\n",
- " dataold = np.copy(old_const['BadPixelsDark'][:, :, :, gain])\n",
+ " if old_const[qm]['BadPixelsDark'] is not None:\n",
+ " dataold = np.copy(old_const[qm]['BadPixelsDark'][:, :, :, gain])\n",
" datau32old = dataold.astype(np.uint32)\n",
" l_data_old.append(len(datau32old[datau32old>0].flatten()))\n",
" for bit in bits:\n",
- " l_data_old.append(np.count_nonzero(old_const['BadPixelsDark'][:, :, :, gain] & bit.value))\n",
+ " l_data_old.append(np.count_nonzero(old_const[qm]['BadPixelsDark'][:, :, :, gain] & bit))\n",
"\n",
" l_data_name = ['All bad pixels', 'NOISE_OUT_OF_THRESHOLD',\n",
" 'OFFSET_OUT_OF_THRESHOLD', 'OFFSET_NOISE_EVAL_ERROR', 'GAIN_THRESHOLDING_ERROR']\n",
@@ -897,7 +942,7 @@
" for i in range(len(l_data)):\n",
" line = [f'{l_data_name[i]}, {gain_names[gain]} gain', l_threshold[i], l_data[i]]\n",
"\n",
- " if old_const['BadPixelsDark'] is not None:\n",
+ " if old_const[qm]['BadPixelsDark'] is not None:\n",
" line += [l_data_old[i]]\n",
" else:\n",
" line += ['-']\n",
@@ -906,8 +951,7 @@
" table.append(['', '', '', ''])\n",
"\n",
"display(Markdown('''\n",
- "\n",
- "### Number of bad pixels ###\n",
+ "### Number of bad pixels\n",
"\n",
"One pixel can be bad for different reasons, therefore, the sum of all types of bad pixels can be more than the number of all bad pixels.\n",
"\n",
@@ -935,49 +979,67 @@
"else:\n",
" constants = ['Offset', 'Noise', 'ThresholdsDark']\n",
"\n",
- "for const in constants:\n",
+ "constants_x_qms = list(itertools.product(constants, res.keys()))\n",
+ "\n",
"\n",
+ "def compute_table(const, qm):\n",
" if const == 'ThresholdsDark':\n",
" table = [['','HG-MG threshold', 'HG-MG threshold', 'MG-LG threshold', 'MG-LG threshold']]\n",
" else:\n",
" table = [['','High gain', 'High gain', 'Medium gain', 'Medium gain', 'Low gain', 'Low gain']]\n",
"\n",
- " for qm in res.keys():\n",
- " data = np.copy(res[qm][const])\n",
+ " compare_with_old_constant = old_const[qm][const] is not None and \\\n",
+ " old_const[qm]['BadPixelsDark'] is not None\n",
+ "\n",
+ " data = np.copy(res[qm][const])\n",
+ "\n",
+ " if const == 'ThresholdsDark':\n",
+ " data[...,0][res[qm]['BadPixelsDark'][...,0]>0] = np.nan\n",
+ " data[...,1][res[qm]['BadPixelsDark'][...,1]>0] = np.nan\n",
+ " else:\n",
+ " data[res[qm]['BadPixelsDark']>0] = np.nan\n",
"\n",
+ " if compare_with_old_constant:\n",
+ " data_old = np.copy(old_const[qm][const])\n",
" if const == 'ThresholdsDark':\n",
- " data[...,0][res[qm]['BadPixelsDark'][...,0]>0] = np.nan\n",
- " data[...,1][res[qm]['BadPixelsDark'][...,1]>0] = np.nan\n",
+ " data_old[...,0][old_const[qm]['BadPixelsDark'][...,0]>0] = np.nan\n",
+ " data_old[...,1][old_const[qm]['BadPixelsDark'][...,1]>0] = np.nan\n",
" else:\n",
- " data[res[qm]['BadPixelsDark']>0] = np.nan\n",
- "\n",
- " if old_const[const] is not None and old_const['BadPixelsDark'] is not None:\n",
- " dataold = np.copy(old_const[const])\n",
- " if const == 'ThresholdsDark':\n",
- " dataold[...,0][old_const['BadPixelsDark'][...,0]>0] = np.nan\n",
- " dataold[...,1][old_const['BadPixelsDark'][...,1]>0] = np.nan\n",
+ " data_old[old_const[qm]['BadPixelsDark']>0] = np.nan\n",
+ "\n",
+ " f_list = [np.nanmedian, np.nanmean, np.nanstd, np.nanmin, np.nanmax]\n",
+ " n_list = ['Median', 'Mean', 'Std', 'Min', 'Max']\n",
+ "\n",
+ " def compute_row(i):\n",
+ " line = [n_list[i]]\n",
+ " for gain in range(3):\n",
+ " # Compare only 3 threshold gain-maps\n",
+ " if gain == 2 and const == 'ThresholdsDark':\n",
+ " continue\n",
+ " stat_measure = f_list[i](data[...,gain])\n",
+ " line.append(f\"{stat_measure:6.1f}\")\n",
+ " if compare_with_old_constant:\n",
+ " old_stat_measure = f_list[i](data_old[...,gain])\n",
+ " line.append(f\"{old_stat_measure:6.1f}\")\n",
" else:\n",
- " dataold[old_const['BadPixelsDark']>0] = np.nan\n",
- "\n",
- " f_list = [np.nanmedian, np.nanmean, np.nanstd, np.nanmin, np.nanmax]\n",
- " n_list = ['Median', 'Mean', 'Std', 'Min', 'Max']\n",
- "\n",
- " for i, f in enumerate(f_list):\n",
- " line = [n_list[i]]\n",
- " for gain in range(3):\n",
- " # Compare only 3 threshold gain-maps\n",
- " if gain == 2 and const == 'ThresholdsDark':\n",
- " continue\n",
- " line.append('{:6.1f}'.format(f(data[...,gain])))\n",
- " if old_const[const] is not None and old_const['BadPixelsDark'] is not None:\n",
- " line.append('{:6.1f}'.format(f(dataold[...,gain])))\n",
- " else:\n",
- " line.append('-')\n",
+ " line.append(\"-\")\n",
+ " return line\n",
+ " \n",
"\n",
- " table.append(line)\n",
+ " with multiprocessing.pool.ThreadPool(processes=multiprocessing.cpu_count() // len(constants_x_qms)) as pool:\n",
+ " rows = pool.map(compute_row, range(len(f_list)))\n",
+ "\n",
+ " table.extend(rows)\n",
+ "\n",
+ " return table\n",
+ "\n",
+ "\n",
+ "with multiprocessing.Pool(processes=len(constants_x_qms)) as pool:\n",
+ " tables = pool.starmap(compute_table, constants_x_qms)\n",
"\n",
- " display(Markdown('### {} [ADU], good pixels only ###'.format(const)))\n",
- " md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header)))"
+ "for (const, qm), table in zip(constants_x_qms, tables):\n",
+ " display(Markdown(f\"### {qm}: {const} [ADU], good pixels only\"))\n",
+ " display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header)))"
]
}
],
diff --git a/notebooks/test/test-cli.ipynb b/notebooks/test/test-cli.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..ef866040aa10b5ebfb5df881f3c493c4f3724652
--- /dev/null
+++ b/notebooks/test/test-cli.ipynb
@@ -0,0 +1,70 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Test Notebook - CLI\n",
+ "\n",
+ "Author: Robert Rosca\n",
+ "\n",
+ "Version: 0.1\n",
+ "\n",
+ "Notebook for use with the unit and continuous integration tests."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/dev/null\" # input folder\n",
+ "out_folder = \"/dev/null\" # output folder\n",
+ "list_normal = [10] # parameterized list, range allowed\n",
+ "list_intellilist = [2345] # parameterized list with ranges, range allowed\n",
+ "concurrency_parameter = [1] # concurrency parameter, range allowed\n",
+ "number = 0 # parameterized number"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Tests notebook execution by just creating an empty file in the output directory."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from pathlib import Path"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = Path(in_folder)\n",
+ "\n",
+ "(in_folder / \"touch\").touch()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.6.8 64-bit",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python",
+ "version": ""
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/pyproject.toml b/pyproject.toml
index 070d67401f955e628c80fe3ca2a46cf8fc972e40..bddced5c9ea33d7f1a437c0b1675c2e8428595f2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,6 +4,12 @@ requires = ["cython==0.29.21", "numpy==1.19.1", "setuptools>=40.8.0", "wheel"]
[tool.isort]
profile = "black"
+[tool.pylint.messages_control]
+disable = "C0330, C0326"
+
+[tool.pylint.format]
+max-line-length = "88"
+
[tool.pytest.ini_options]
norecursedirs = [
"legacy",
@@ -15,5 +21,11 @@ norecursedirs = [
"dist",
"node_modules",
"venv",
- "{arch}"
+ "{arch}",
+]
+
+required_plugins = [
+ "pytest-asyncio",
+ "pytest-cov",
+ "pytest-subprocess",
]
diff --git a/setup.py b/setup.py
index 358f08e86d61a58555c7edfd77733452e74a9095..7a65bbfa5b7dc6d52ebce011678d601677284a85 100644
--- a/setup.py
+++ b/setup.py
@@ -74,14 +74,14 @@ setup(
ext_modules=cythonize(ext_modules),
install_requires=[
"iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.0.7", # noqa
- "nbparameterise @ git+ssh://git@git.xfel.eu:10022/detectors/nbparameterise.git@0.3", # noqa
"XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.5.6-2.10.0#subdirectory=lib", # noqa
+ "pasha @ git+https://github.com/European-XFEL/pasha.git@0.1.0",
"Cython==0.29.21",
"Jinja2==2.11.2",
"astcheck==0.2.5",
"astsearch==0.1.3",
"dill==0.3.0",
- "extra_data==1.2.0",
+ "extra_data==1.4.1",
"extra_geom==1.1.1",
"fabio==0.9.0",
"gitpython==3.1.0",
@@ -101,6 +101,7 @@ setup(
"nbclient==0.5.1",
"nbconvert==5.6.1",
"nbformat==5.0.7",
+ "nbparameterise==0.5",
"notebook==6.1.5",
"numpy==1.19.1",
"prettytable==0.7.2",
@@ -125,6 +126,7 @@ setup(
"nbval",
"pytest-asyncio",
"pytest-cov",
+ "pytest-subprocess",
"pytest>=5.4.0",
"testpath",
"unittest-xml-reporting==3.0.2",
diff --git a/src/xfel_calibrate/calibrate.py b/src/xfel_calibrate/calibrate.py
index e0e43e1f71d29598e73febd050b4e1c5f16cbccf..951dba8ea94b711cff5ce1796b7486633696f107 100755
--- a/src/xfel_calibrate/calibrate.py
+++ b/src/xfel_calibrate/calibrate.py
@@ -15,16 +15,28 @@ from pathlib import Path
from subprocess import DEVNULL, check_output
from typing import List, Union
-import cal_tools.tools
-import nbconvert
import nbformat
import numpy as np
from jinja2 import Template
from nbparameterise import extract_parameters, parameter_values, replace_definitions
+import cal_tools.tools
+
from .finalize import tex_escape
from .notebooks import notebooks
-from .settings import *
+from .settings import (
+ default_report_path,
+ free_nodes_cmd,
+ launcher_command,
+ max_reserved,
+ preempt_nodes_cmd,
+ python_path,
+ reservation,
+ reservation_char,
+ sprof,
+ temp_path,
+ try_report_to_output,
+)
PKG_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -115,7 +127,6 @@ def make_intelli_list(ltype):
super(IntelliListAction, self).__init__(*args, **kwargs)
def __call__(self, parser, namespace, values, option_string=None):
-
parsed_values = []
values = ",".join(values)
if isinstance(values, str):
@@ -171,7 +182,7 @@ def extract_title_author_version(nb):
# In case of a standard installation a version is stored
# in the _version.py file
try:
- git_dir = os.path.join(PKG_DIR, '..', '.git')
+ git_dir = os.path.join(PKG_DIR, '..', '..', '.git')
version = check_output([
'git', f'--git-dir={git_dir}', 'describe', '--tag',
], stderr=DEVNULL).decode('utf8')
@@ -284,8 +295,7 @@ def balance_sequences(in_folder: str, run: int, sequences: List[int],
if isinstance(karabo_da, str):
karabo_da = [karabo_da]
elif not isinstance(karabo_da, list):
- raise ValueError("Balance sequences expects "
- "karabo_da as a string or list.")
+ raise TypeError("Balance sequences expects `karabo_da` as a string or list.")
in_path = Path(in_folder, f"r{run:04d}")
@@ -305,8 +315,10 @@ def balance_sequences(in_folder: str, run: int, sequences: List[int],
if sequences != [-1]:
seq_nums = sorted(seq_nums.intersection(sequences))
if len(seq_nums) == 0:
- raise ValueError(f"Selected sequences {sequences} are not "
- f"available in {in_path}")
+ raise ValueError(
+ f"Selected sequences {sequences} are not "
+ f"available in {in_path}"
+ )
# Validate required nodes with max_nodes
nsplits = len(seq_nums) // sequences_per_node
@@ -332,6 +344,7 @@ def make_extended_parser() -> argparse.ArgumentParser:
try:
det_notebooks = notebooks[detector]
except KeyError:
+ # TODO: This should really go to stderr not stdout
print("Not one of the known detectors: {}".format(notebooks.keys()))
sys.exit(1)
@@ -394,7 +407,7 @@ def add_args_from_nb(nb, parser, cvar=None, no_required=False):
:param bool no_required: If True, none of the added options are required.
"""
parser.description = make_epilog(nb)
- parms = extract_parameters(nb, lang='python3')
+ parms = extract_parameters(nb, lang='python')
for p in parms:
@@ -663,7 +676,7 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
suffix = flatten_list(cval)
# first convert the notebook
- parms = extract_parameters(nb, lang='python3')
+ parms = extract_parameters(nb, lang='python')
if has_parm(parms, "cluster_profile"):
cluster_profile = f"{args['cluster_profile']}_{suffix}"
@@ -673,7 +686,7 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
params = parameter_values(parms, **args)
params = parameter_values(params, cluster_profile=cluster_profile)
- new_nb = replace_definitions(nb, params, execute=False, lang='python3')
+ new_nb = replace_definitions(nb, params, execute=False, lang='python')
if not show_title:
first_markdown_cell(new_nb).source = ''
set_figure_format(new_nb, args["vector_figs"])
@@ -682,9 +695,7 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
os.path.basename(base_name), cparm, suffix)
nbpath = os.path.join(temp_path, new_name)
- with open(nbpath, "w") as f:
- f.write(nbconvert.exporters.export(
- nbconvert.NotebookExporter, new_nb)[0])
+ nbformat.write(new_nb, nbpath)
# add finalization to the last job
if final_job:
@@ -832,7 +843,7 @@ def run():
if ext_func is not None:
extend_params(nb, ext_func)
- parms = extract_parameters(nb, lang='python3')
+ parms = extract_parameters(nb, lang='python')
title, author, version = extract_title_author_version(nb)
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index 5f235bcea92daa0847033697a7020a8d912827ef..a98aeef536ec49d0a51cccb2abe789203c1c935a 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -239,4 +239,14 @@ notebooks = {
"cluster cores": 16},
},
},
+ "TEST": {
+ "TEST-CLI": {
+ "notebook": "notebooks/test/test-cli.ipynb",
+ "concurrency": {
+ "parameter": "concurrency_parameter",
+ "default concurrency": None,
+ "cluster cores": 1,
+ },
+ },
+ },
}
diff --git a/src/xfel_calibrate/settings.py b/src/xfel_calibrate/settings.py
index 709fdd5ec7c61fa01b3dca68112fd31e36409c1c..43e79c9a163e4c42dc4a66ceee43ca33d929d5f3 100644
--- a/src/xfel_calibrate/settings.py
+++ b/src/xfel_calibrate/settings.py
@@ -17,7 +17,7 @@ try_report_to_output = True
logo_path = "xfel.pdf"
# the command to run this concurrently. It is prepended to the actual call
-sprof = os.environ.get("XFELCALSLURM", "exfel")
+sprof = os.environ.get("XFELCALSLURM", "exfel") # TODO: https://git.xfel.eu/gitlab/calibration/planning/issues/3
launcher_command = "sbatch -t 24:00:00 --requeue --output {temp_path}/slurm-%j.out"
free_nodes_cmd = "sinfo -p exfel -t idle -N --noheader | wc -l"
preempt_nodes_cmd = "squeue -p all,grid --noheader | grep max-exfl | egrep -v 'max-exfl18[3-8]|max-exfl100|max-exflg' | wc -l"
diff --git a/src/xfel_calibrate/titlepage.tmpl b/src/xfel_calibrate/titlepage.tmpl
index 39c8e8ac6be4647c0df260bd00f77cd5ac4e7381..31c0f08ad7089acc57623c73317ec09b0c25d484 100644
--- a/src/xfel_calibrate/titlepage.tmpl
+++ b/src/xfel_calibrate/titlepage.tmpl
@@ -10,7 +10,7 @@
\end{figure}
\vspace{0mm}
- \Large \textbf{ Detector group }
+ \Large \textbf{ Data department }
Based on data sample: {{ data_path }}
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..d0e461a4cf88ab616b5ca7f92ac25586efb7c1eb
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,25 @@
+from pathlib import Path
+
+import pytest
+
+
+def pytest_addoption(parser):
+ parser.addoption(
+ "--no-gpfs",
+ action="store_true",
+ default="false",
+ help="Skips tests marked as requiring GPFS access",
+ )
+
+
+def pytest_configure(config):
+ config.addinivalue_line(
+ "markers", "requires_gpfs(): marks skips for tests that require GPFS access"
+ )
+
+
+def pytest_runtest_setup(item):
+ if list(item.iter_markers(name="requires_gpfs")) and (
+ not Path("/gpfs").is_dir() or item.config.getoption("--no-gpfs")
+ ):
+ pytest.skip("gpfs not available")
diff --git a/tests/test_cal_tools.py b/tests/test_cal_tools.py
index 7ee13b577056da2017de34bd4a684d01d0fa120b..62567e43832923aa1d2b39b5491b56106f4c40ec 100644
--- a/tests/test_cal_tools.py
+++ b/tests/test_cal_tools.py
@@ -21,6 +21,7 @@ def test_show_processed_modules():
assert 'LDP' in err.value()
+@pytest.mark.requires_gpfs
def test_dir_creation_date():
folder = '/gpfs/exfel/exp/CALLAB/202031/p900113/raw'
@@ -54,46 +55,48 @@ cal_db_interface = "tcp://max-exfl017:8020"
def test_get_pdu_from_db():
+ snapshot_at = "2021-05-06 00:20:10.00"
+
# A karabo_da str returns a list of one element.
- pdu_dict = get_pdu_from_db(karabo_id="TEST_DET_CI-2",
- karabo_da="TEST_DAQ_DA_01",
+ pdu_dict = get_pdu_from_db(karabo_id="TEST_DET_CAL_CI-1",
+ karabo_da="TEST_DET_CAL_DA0",
constant=constant,
condition=condition,
cal_db_interface=cal_db_interface,
- snapshot_at="2021-03-01 09:44:00+00:00",
+ snapshot_at=snapshot_at,
timeout=30000)
assert len(pdu_dict) == 1
- assert pdu_dict[0] == 'PHYSICAL_DETECTOR_UNIT-1_DO_NOT_DELETE'
+ assert pdu_dict[0] == "CAL_PHYSICAL_DETECTOR_UNIT-1_TEST"
# A list of karabo_das to return thier PDUs, if available.
- pdu_dict = get_pdu_from_db(karabo_id="TEST_DET_CI-2",
- karabo_da=["TEST_DAQ_DA_01", "TEST_DAQ_DA_02",
+ pdu_dict = get_pdu_from_db(karabo_id="TEST_DET_CAL_CI-1",
+ karabo_da=["TEST_DET_CAL_DA0", "TEST_DET_CAL_DA1",
"UNAVAILABLE_DA"],
constant=constant,
condition=condition,
cal_db_interface=cal_db_interface,
- snapshot_at="2021-03-01 09:44:00+00:00",
+ snapshot_at=snapshot_at,
timeout=30000)
- assert pdu_dict == ['PHYSICAL_DETECTOR_UNIT-1_DO_NOT_DELETE',
- 'PHYSICAL_DETECTOR_UNIT-2_DO_NOT_DELETE',
+ assert pdu_dict == ["CAL_PHYSICAL_DETECTOR_UNIT-1_TEST",
+ "CAL_PHYSICAL_DETECTOR_UNIT-2_TEST",
None]
# "all" is used to return all corresponding units for a karabo_id.
- pdu_dict = get_pdu_from_db(karabo_id="TEST_DET_CI-2",
+ pdu_dict = get_pdu_from_db(karabo_id="TEST_DET_CAL_CI-1",
karabo_da="all",
constant=constant,
condition=condition,
- cal_db_interface="tcp://max-exfl017:8020",
- snapshot_at="2021-03-01 09:44:00+00:00",
+ cal_db_interface=cal_db_interface,
+ snapshot_at=snapshot_at,
timeout=30000)
- assert len(pdu_dict) == 3
- assert pdu_dict == ['PHYSICAL_DETECTOR_UNIT-1_DO_NOT_DELETE',
- 'PHYSICAL_DETECTOR_UNIT-2_DO_NOT_DELETE',
- 'PHYSICAL_DETECTOR_UNIT-3_DO_NOT_DELETE']
+ assert len(pdu_dict) == 2
+ assert pdu_dict == ["CAL_PHYSICAL_DETECTOR_UNIT-1_TEST",
+ "CAL_PHYSICAL_DETECTOR_UNIT-2_TEST"]
+@pytest.mark.requires_gpfs
def test_initialize_from_db():
creation_time = datetime.strptime("2020-01-07 13:26:48.00",
"%Y-%m-%d %H:%M:%S.%f")
diff --git a/tests/test_calibrate.py b/tests/test_calibrate.py
deleted file mode 100644
index 7cc48d61b9ea84a98aae7e34da80a246612af605..0000000000000000000000000000000000000000
--- a/tests/test_calibrate.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import pytest
-
-from xfel_calibrate.calibrate import balance_sequences
-
-
-def test_balance_sequences():
-
- ret = balance_sequences(in_folder="/gpfs/exfel/exp/CALLAB/202031/p900113/raw", # noqa
- run=9992, sequences=[0, 2, 5, 10, 20, 50, 100],
- sequences_per_node=1, karabo_da=["all"],
- max_nodes=8)
-
- expected = [[0], [2]]
- assert expected == ret
-
- ret = balance_sequences(in_folder="/gpfs/exfel/exp/CALLAB/202031/p900113/raw", # noqa
- run=9992, sequences=[-1],
- sequences_per_node=1, karabo_da=["JNGFR01"],
- max_nodes=3)
- expected = []
- assert expected == ret
-
- with pytest.raises(ValueError) as e:
- balance_sequences(in_folder="/gpfs/exfel/exp/CALLAB/202031/p900113/raw", # noqa
- run=9992, sequences=[1991, 2021],
- sequences_per_node=1, karabo_da=["all"],
- max_nodes=3)
- assert 'Selected sequences [1991, 2021]]' in e.value()
-
- with pytest.raises(ValueError) as e:
- balance_sequences(in_folder="/gpfs/exfel/exp/CALLAB/202031/p900113/raw", # noqa
- run=9992, sequences=[1991, 2021],
- sequences_per_node=1, karabo_da=-1,
- max_nodes=3)
- assert 'karabo_da as a string or list' in e.value()
diff --git a/tests/test_webservice.py b/tests/test_webservice.py
index 604668a93b4db1365ff1c0773866451bf122373e..f26ea60d9ce2cae40c4665393b0cba272b6c50d0 100644
--- a/tests/test_webservice.py
+++ b/tests/test_webservice.py
@@ -9,6 +9,7 @@ sys.path.insert(0, Path(__file__).parent / 'webservice')
from webservice.webservice import check_files, merge, parse_config, wait_on_transfer
+@pytest.mark.requires_gpfs
def test_check_files():
in_folder = '/gpfs/exfel/exp/CALLAB/202031/p900113/raw'
runs = [9985, 9984]
diff --git a/tests/test_xfel_calibrate/__init__.py b/tests/test_xfel_calibrate/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/tests/test_xfel_calibrate/conftest.py b/tests/test_xfel_calibrate/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..c00cfda97c78621930ce5fe2797f94b66ed9068e
--- /dev/null
+++ b/tests/test_xfel_calibrate/conftest.py
@@ -0,0 +1,240 @@
+# pylint: disable=missing-module-docstring
+import sys
+from pathlib import Path
+from typing import Callable, Dict, List, NamedTuple, Optional, Union
+from unittest import mock
+
+import extra_data.tests.make_examples as make_examples
+from pytest_subprocess import FakeProcess
+
+from xfel_calibrate import calibrate, settings
+
+
+class FakeProcessCalibrate(FakeProcess):
+ """Class used to create a fake process which will mock calls to processes
+ expected to be called by pycalibration
+ """
+ def __init__(self):
+ """Sets up fake process for use with xfel calibrate
+
+ Fakes:
+ - Slurm free nodes call
+ - Slurm preempt nodes call
+ - Slurm sbatch call
+ - Git describe calls (for pulling repo version)
+
+ Expected use is something like:
+
+ ```
+ @pytest.fixture(scope="class", autouse=True)
+ def fake_process_calibrate(self):
+ with FakeProcessCalibrate() as fake_process:
+ yield fake_process
+ ```
+ """
+ super().__init__()
+ # Fake calls to slurm
+ self.register_subprocess(settings.free_nodes_cmd, stdout=["1"])
+ self.register_subprocess(settings.preempt_nodes_cmd, stdout=["1"])
+ self.register_subprocess(
+ ["sbatch", self.any()], stdout=["Submitted batch job 000000"]
+ )
+
+ # For the version insertion...
+ self.register_subprocess(
+ ["git", self.any(), "describe", "--tag"], stdout=["0.0.0"]
+ )
+
+ self.keep_last_process(True)
+
+
+class MockProposal:
+ """Class used with pytest to create proposal directories with placeholder
+ files
+
+ Use this to generate a skeleton of the proposal structure for tests which
+ only rely on the files being present and not their contents, e.g. balancing
+ by sequences done in some notebooks
+ """
+
+ def __init__(
+ self,
+ *,
+ tmp_path: Path,
+ runs: Union[int, list, slice],
+ instrument: str,
+ cycle: str,
+ proposal: str,
+ sequences: int = 2,
+ ):
+ """Create a MockProposal object, this should be used to create a fixture
+
+ Expected use is something like:
+
+ ```python3
+ @pytest.fixture
+ def mock_proposal(tmp_path):
+ return MockProposal(tmp_path, runs=4)
+ ```
+
+ Then `mock_proposal` can be passed as a fixture (or set to autouse) to
+ make the object available for individual or class tests
+
+ Args:
+ tmp_path (Path): Temporary path, should come from pytest fixture
+ runs (Union[int, list, slice]): Defines what run directories to make
+ instrument (str, optional): Instrument name, e.g. "AGIPD".
+ cycle (str, optional): Cycle, e.g. "202031".
+ proposal (str, optional): Proposal number, e.g. "p900113".
+ sequences (int, optional): Number of sequence files. Defaults to 2.
+ """
+ self.tmp_path = tmp_path
+ self.instrument = instrument
+ self.cycle = cycle
+ self.proposal = proposal
+ self.sequences = list(
+ range(sequences)
+ ) # TODO: Implement once it's in extra-data
+
+ self.path = tmp_path / instrument / cycle / proposal
+ self.path_raw = self.path / "raw"
+ self.path_proc = self.path / "proc"
+
+ self.runs = self.create_runs(runs)
+
+ def create_runs(self, runs: Union[int, List[int], slice]) -> Dict[int, Path]:
+ """Create run directories with skeleton files for the proposal
+
+ Args:
+ runs (Union[int, list, slice]): Defines what run directories to make
+
+ Returns:
+ [Dict[int, Path]]: Dictionary of the run number and run directory
+ """
+ if isinstance(runs, int):
+ runs = list(range(runs))
+ elif isinstance(runs, list):
+ if not all(isinstance(r, int) for r in runs):
+ raise TypeError("lists of runs must contain only integers")
+ elif isinstance(runs, slice):
+ if runs.stop:
+ runs = list(range(runs.stop)[runs])
+ else:
+ raise ValueError("runs slice must have a stop value")
+ else:
+ raise TypeError("runs must be an int, list of integers, or a slice")
+
+ run_directories = {run: self.path_raw / f"r{run:04d}" for run in runs}
+
+ for run_path in run_directories.values():
+ run_path.mkdir(parents=True)
+
+ self._create_run(run_path)
+
+ return run_directories
+
+ @property
+ def _create_run(self) -> Callable:
+ """Return the number of modules for a detector type
+
+ Returns:
+ modules [Iterable]: Number of modules for a detector type
+ """
+ return {
+ "AGIPD": make_examples.make_spb_run,
+ "DSSC": make_examples.make_scs_run,
+ "JGFR": make_examples.make_jungfrau_run,
+ "LPD": make_examples.make_fxe_run,
+ "None": lambda x: None,
+ }[self.instrument]
+
+
+class CalibrateCall:
+ """Class used with pytest to create call `xfel-calibrate` and log its output
+
+ Use this to manage calls to `xfel-calibrate` via pytest fixtures
+ """
+
+ def __init__(
+ self,
+ tmp_path,
+ capsys,
+ *,
+ detector: str,
+ cal_type: str,
+ command: str = "xfel-calibrate",
+ in_folder: Optional[Path] = None,
+ out_folder: Optional[Path] = None,
+ extra_args: List[str] = None,
+ ):
+ """Create a CallibrateCall object, this should be used to create a fixture
+
+ Expected use is something like:
+
+ ```python3
+ @pytest.fixture(scope="function")
+ def calibrate_call(self, mock_proposal: MockProposal, capsys, tmp_path):
+ return CalibrateCall(
+ tmp_path,
+ capsys,
+ in_folder=mock_proposal.path_raw,
+ out_folder=mock_proposal.path_proc,
+ command="xfel-calibrate",
+ detector="AGIPD",
+ cal_type="CORRECT",
+ extra_args=["--run", "0"],
+ )
+ ```
+
+ Args:
+ tmp_path ([type]): Temporary path, should come from pytest fixture
+ capsys ([type]): capsys path, should come from pytest fixture
+ detector (str): Detector passed to the command, e.g. AGIPD
+ cal_type (str): Calibration type passed to the command, e.g. CORRECT
+ command (str): Main entrypoint called. Defaults to "xfel-calibrate".
+ in_folder (Optional[Path], optional): Path to the input folder, usually
+ raw. Defaults to None.
+ out_folder (Optional[Path], optional): Path to the output folder, usually
+ proc. Defaults to None.
+ extra_args (List[str], optional): Additional arguments to pass to the
+ command. Defaults to None.
+ """
+ self.tmp_path = tmp_path
+ self.in_folder = in_folder
+ self.out_folder = out_folder
+
+ self.args = []
+ self.args.extend([command, detector, cal_type])
+ if in_folder:
+ self.args.extend(["--in-folder", str(self.in_folder)])
+ if out_folder:
+ self.args.extend(["--out-folder", str(self.out_folder)])
+
+ if extra_args:
+ self.args.extend(extra_args)
+
+ with mock.patch.object(sys, "argv", self.args):
+ with mock.patch.object(calibrate, "temp_path", str(tmp_path)):
+ calibrate.run()
+
+ out, err = capsys.readouterr()
+
+ self.out: str = out
+ self.err: str = err
+
+ Paths = NamedTuple(
+ "Paths",
+ [
+ ("notebooks", List[Path]),
+ ("run_calibrate", Path),
+ ("finalize", Path),
+ ("InputParameters", Path),
+ ],
+ )
+
+ self.paths = Paths(
+ notebooks=list(self.tmp_path.glob("**/*/*.ipynb")),
+ run_calibrate=list(self.tmp_path.glob("**/*/run_calibrate.sh"))[0],
+ finalize=list(self.tmp_path.glob("**/*/finalize.py"))[0],
+ InputParameters=list(self.tmp_path.glob("**/*/InputParameters.rst"))[0],
+ )
diff --git a/tests/test_xfel_calibrate/test_calibrate.py b/tests/test_xfel_calibrate/test_calibrate.py
new file mode 100644
index 0000000000000000000000000000000000000000..b290cf12ab05c35359d2e2ca2de98e1d902b5c15
--- /dev/null
+++ b/tests/test_xfel_calibrate/test_calibrate.py
@@ -0,0 +1,52 @@
+# TODO: These are unit tests, `test_cli.py` contains integration tests, may be
+# worth splitting these up in the future so that it's easier to track
+# what's-what, track the coverage of both approaches individually, and run them
+# independently from each other
+
+import pytest
+
+from xfel_calibrate.calibrate import balance_sequences
+
+
+@pytest.mark.parametrize(
+ "karabo_da,sequences,expected",
+ [
+ pytest.param(
+ "all",
+ [0, 2, 5, 10, 20, 50, 100],
+ [[0], [2]],
+ marks=pytest.mark.requires_gpfs(),
+ ),
+ ("JNGFR01", [-1], []),
+ ],
+)
+def test_balance_sequences(karabo_da, sequences, expected):
+ ret = balance_sequences(
+ in_folder="/gpfs/exfel/exp/CALLAB/202031/p900113/raw",
+ run=9992,
+ sequences=sequences,
+ sequences_per_node=1,
+ karabo_da=karabo_da,
+ max_nodes=8,
+ )
+
+ assert ret == expected
+
+
+@pytest.mark.parametrize(
+ "karabo_da,sequences,expected",
+ [
+ pytest.param("all", [1991, 2021], ValueError, marks=pytest.mark.requires_gpfs()),
+ (-1, [], TypeError),
+ ],
+)
+def test_balance_sequences_raises(karabo_da, sequences, expected):
+ with pytest.raises(expected):
+ balance_sequences(
+ in_folder="/gpfs/exfel/exp/CALLAB/202031/p900113/raw",
+ run=9992,
+ sequences=sequences,
+ sequences_per_node=1,
+ karabo_da=karabo_da,
+ max_nodes=8,
+ )
diff --git a/tests/test_xfel_calibrate/test_cli.py b/tests/test_xfel_calibrate/test_cli.py
new file mode 100644
index 0000000000000000000000000000000000000000..f0ffeb4af21ededff97abe66e22f669266517e6d
--- /dev/null
+++ b/tests/test_xfel_calibrate/test_cli.py
@@ -0,0 +1,408 @@
+# pylint: disable=missing-class-docstring, missing-function-docstring, no-self-use
+"""Tests for the CLI portion of `xfel_calibrate`
+
+These tests cover the CLI interface which is called by the `xfel-calibrate ...`
+entrypoint. Some sections of the `calibrate.py` file are still not covered by
+the current test cases, this should be improved later on.
+"""
+
+import ast
+import shlex
+import sys
+from datetime import date
+from pathlib import Path
+from unittest import mock
+
+import nbformat
+import pytest
+from nbparameterise import extract_parameters
+
+import xfel_calibrate.calibrate as calibrate
+from tests.test_xfel_calibrate.conftest import (
+ CalibrateCall,
+ FakeProcessCalibrate,
+ MockProposal,
+)
+
+
+class TestBasicCalls:
+ """Tests which only call the command line utility `xfel-calibrate` and check
+ that the expected output is present in stdout
+ """
+
+ @mock.patch.object(sys, "argv", ["xfel-calibrate", "--help"])
+ def test_help(self, capsys):
+ with pytest.raises(SystemExit):
+ calibrate.run()
+
+ out, err = capsys.readouterr()
+
+ # Should always be present in these help outputs
+ assert "positional arguments:" in out
+ assert "optional arguments:" in out
+
+ assert err == ""
+
+ @mock.patch.object(sys, "argv", ["xfel-calibrate", "TEST", "-h"])
+ def test_help_detector(self, capsys):
+ with pytest.raises(SystemExit):
+ calibrate.run()
+
+ out, err = capsys.readouterr()
+
+ assert "Notebook for use with the unit and continuous integration" in out
+ assert "tests." in out
+
+ assert err == ""
+
+ @mock.patch.object(sys, "argv", ["xfel-calibrate", "NotADetector", "beep", "-h"])
+ def test_unknown_detector(self, capsys):
+ with pytest.raises(SystemExit) as exit_exception:
+ calibrate.run()
+
+ out, err = capsys.readouterr()
+
+ assert exit_exception.value.code == 1
+
+ assert "Not one of the known calibrations or detectors" in out
+
+ assert err == ""
+
+ @mock.patch.object(sys, "argv", ["xfel-calibrate", "NotADetector", "-h"])
+ def test_unknown_detector_h(self, capsys):
+ with pytest.raises(SystemExit) as exit_exception:
+ calibrate.run()
+
+ out, err = capsys.readouterr()
+
+ assert exit_exception.value.code == 1
+
+ assert "Not one of the known detectors" in out
+
+ assert err == ""
+
+ @mock.patch.object(sys, "argv", ["xfel-calibrate", "Tutorial", "TEST", "--help"])
+ def test_help_nb(self, capsys):
+ with pytest.raises(SystemExit):
+ calibrate.run()
+
+ out, err = capsys.readouterr()
+
+ # Should always be present in these help outputs
+ assert "positional arguments:" in out
+ assert "optional arguments:" in out
+
+ # Defined in the test notebook, should be propagated to help output
+ assert "sensor-size" in out
+ assert "random-seed" in out
+
+ assert err == ""
+
+
+class TestTutorialNotebook:
+ """Checks calling `xfel-calibrate` on the `Tutorial TEST` notebook, looks
+ at the stdout as well as files generated by the call
+ """
+
+ @pytest.fixture(scope="class", autouse=True)
+ def fake_process_calibrate(self):
+ with FakeProcessCalibrate() as fake_process:
+ yield fake_process
+
+ @pytest.fixture(scope="class")
+ def mock_proposal(self, tmp_path_factory):
+ return MockProposal(
+ tmp_path=tmp_path_factory.mktemp("exp"),
+ instrument="None",
+ cycle="000000",
+ proposal="p000000",
+ runs=0,
+ sequences=0,
+ )
+
+ @pytest.fixture(scope="function")
+ def calibrate_call(
+ self,
+ mock_proposal: MockProposal,
+ capsys,
+ tmp_path,
+ ):
+ return CalibrateCall(
+ tmp_path,
+ capsys,
+ out_folder=mock_proposal.path_proc,
+ detector="Tutorial",
+ cal_type="TEST",
+ extra_args=["--runs", "1000"],
+ )
+
+ def test_call(
+ self,
+ calibrate_call: CalibrateCall,
+ ):
+ assert "sbatch" in calibrate_call.out
+ assert "--job-name xfel_calibrate" in calibrate_call.out
+ assert str(calibrate_call.tmp_path) in calibrate_call.out
+ assert "Submitted job: " in calibrate_call.out
+
+ assert calibrate_call.err == ""
+
+ def test_expected_processes_called(
+ self,
+ fake_process_calibrate: FakeProcessCalibrate,
+ ):
+ process_calls = [
+ list(shlex.shlex(p, posix=True, punctuation_chars=True))
+ if isinstance(p, str)
+ else p
+ for p in fake_process_calibrate.calls
+ ]
+
+ processes_called = [p[0] for p in process_calls] # List of the process names
+
+ assert "sbatch" in processes_called
+
+ @pytest.mark.skip(reason="not implemented")
+ def test_output_metadata_yml(self):
+ # TODO: Finish this test later, not a priority
+ # metadata_yml_path = list(self.tmp_path.glob("**/calibration_metadata.yml"))
+ pass
+
+ def test_output_ipynb(self, calibrate_call: CalibrateCall):
+ notebook_path = calibrate_call.paths.notebooks
+
+ assert len(notebook_path) == 1
+
+ with notebook_path[0].open() as file:
+ notebook = nbformat.read(file, as_version=4)
+
+ parameters = {p.name: p.value for p in extract_parameters(notebook)}
+
+ assert parameters["out_folder"] == str(calibrate_call.out_folder)
+ assert parameters["sensor_size"] == [10, 30]
+ assert parameters["random_seed"] == [2345]
+ assert parameters["runs"] == 1000
+
+ def test_output_finalize(
+ self, mock_proposal: MockProposal, calibrate_call: CalibrateCall
+ ):
+ # TODO: Specify `feature_version` once we run on python 3.8+
+ finalize_ast = ast.parse(calibrate_call.paths.finalize.read_text())
+
+ today = date.today()
+
+ expected_equals = {
+ "joblist": [],
+ "project": "Tutorial Calculation",
+ "calibration": "Tutorial Calculation",
+ "author": "Astrid Muennich",
+ "version": "0.0.0",
+ "data_path": "",
+ }
+
+ expected_contains = {
+ "request_time": str(today),
+ "submission_time": str(today),
+ "run_path": str(calibrate_call.tmp_path),
+ # TODO: add a test checking that the out folder is correct
+ # reffer to: https://git.xfel.eu/gitlab/detectors/pycalibration/issues/52
+ "out_path": str(mock_proposal.path_proc),
+ "report_to": str(mock_proposal.path_proc),
+ }
+
+ # Pull the keyword arguments out of the finalize function call via the AST,
+ # here we use the keys in `expected_...` to filter which kwargs are parsed
+ # as some cannot be read
+ finalize_kwargs = {
+ k.arg: ast.literal_eval(k.value)
+ for k in ast.walk(finalize_ast)
+ if isinstance(k, ast.keyword)
+ and (k.arg in expected_equals or k.arg in expected_contains)
+ }
+
+ for k, v in expected_equals.items():
+ assert v == finalize_kwargs[k]
+
+ for k, v in expected_contains.items():
+ assert v in finalize_kwargs[k]
+
+ @pytest.mark.skip(reason="not implemented")
+ def test_output_rst(self, calibrate_call: CalibrateCall):
+ # TODO: Finish this test later, not a priority
+ # rst_path = calibrate_call.paths.InputParameters
+ pass
+
+ def test_output_sh(self, calibrate_call: CalibrateCall):
+ cmd = list(
+ shlex.shlex(
+ calibrate_call.paths.run_calibrate.read_text(),
+ posix=True,
+ punctuation_chars=True,
+ )
+ )
+
+ assert (
+ cmd[0] == "xfel-calibrate"
+ ), f"{calibrate_call.paths.run_calibrate} does not call `xfel-calibrate`"
+
+ assert cmd[1:3] == ["Tutorial", "TEST"]
+ assert {"--out-folder", str(calibrate_call.out_folder)}.issubset(cmd)
+ assert {"--runs", "1000"}.issubset(cmd)
+
+
+class TestIntelliList:
+ @pytest.fixture(scope="class", autouse=True)
+ def fake_process_calibrate(self):
+ with FakeProcessCalibrate() as fake_process:
+ yield fake_process
+
+ @pytest.fixture(scope="class")
+ def mock_proposal(self, tmpdir_factory):
+ return MockProposal(
+ tmp_path=Path(tmpdir_factory.mktemp("exp")),
+ instrument="AGIPD",
+ cycle="202031",
+ proposal="p900113",
+ runs=1,
+ sequences=1,
+ )
+
+ @pytest.fixture(scope="function")
+ def calibrate_call(self, mock_proposal: MockProposal, capsys, tmp_path):
+ return CalibrateCall(
+ tmp_path,
+ capsys,
+ in_folder=mock_proposal.path_raw,
+ out_folder=mock_proposal.path_proc,
+ detector="TEST",
+ cal_type="TEST-CLI",
+ extra_args=[
+ "--number",
+ "10",
+ "--list-normal",
+ "1,2,10",
+ "--list-intellilist",
+ "1,2,5-8",
+ "--concurrency-parameter",
+ "0,1",
+ ],
+ )
+
+ def test_intellilist(self, calibrate_call: CalibrateCall):
+ assert "--number" in calibrate_call.args
+ assert "--list-intellilist" in calibrate_call.args
+ assert "1,2,5-8" in calibrate_call.args
+
+ assert len(calibrate_call.paths.notebooks) == 2
+
+ for i, notebook_path in enumerate(calibrate_call.paths.notebooks):
+ with notebook_path.open() as file:
+ notebook = nbformat.read(file, as_version=4)
+
+ parameters = {p.name: p.value for p in extract_parameters(notebook)}
+
+ assert parameters["number"] == 10
+ assert parameters["list_normal"] == [1, 2, 10]
+ assert parameters["list_intellilist"] == [1, 2, 5, 6, 7]
+ assert parameters["concurrency_parameter"][0] == i
+
+
+class TestAgipdNotebook:
+ @pytest.fixture(scope="class", autouse=True)
+ def fake_process_calibrate(self):
+ with FakeProcessCalibrate() as fake_process:
+ yield fake_process
+
+ @pytest.fixture(scope="function")
+ def mock_proposal(self, tmpdir_factory):
+ return MockProposal(
+ tmp_path=Path(tmpdir_factory.mktemp("exp")),
+ instrument="AGIPD",
+ cycle="202031",
+ proposal="p900113",
+ runs=2,
+ # TODO: update this once extra-data tests can have variable sequences
+ # sequences=5,
+ )
+
+ @pytest.fixture(scope="function")
+ def calibrate_call(self, mock_proposal: MockProposal, capsys, tmp_path):
+ return CalibrateCall(
+ tmp_path,
+ capsys,
+ in_folder=mock_proposal.path_raw,
+ out_folder=mock_proposal.path_proc / "r0000",
+ detector="AGIPD",
+ cal_type="CORRECT",
+ extra_args=[
+ "--run",
+ "0",
+ "--sequences",
+ "1-3",
+ # TODO: enable this when notebook execution tests are ready to be ran
+ # "--no-cluster-job",
+ ],
+ )
+
+ @pytest.mark.skip(reason="not implemented")
+ def test_out_folder_correct(self):
+ # TODO: add a test checking that the out folder is correct
+ # reffer to: https://git.xfel.eu/gitlab/detectors/pycalibration/issues/52
+ pass
+
+ @pytest.mark.skip(reason="requires extra-data test file sequence options")
+ def test_files_present(self, calibrate_call: CalibrateCall):
+ # There should be three notebooks: one pre, then the main, then one post
+ assert len(calibrate_call.paths.notebooks) == 3
+ # This is pretty fragile, but the name of notebooks should not change
+ # (too) often
+ root_nb_path = calibrate_call.paths.notebooks[0].parent
+ notebooks = [
+ root_nb_path / "AGIPD_Correct_and_Verify__sequences__1.ipynb",
+ root_nb_path / "AGIPD_Correct_and_Verify_Summary_NBC__None__None.ipynb",
+ root_nb_path / "AGIPD_Retrieve_Constants_Precorrection__None__None.ipynb",
+ ]
+
+ assert all(nb in calibrate_call.paths.notebooks for nb in notebooks)
+
+ @pytest.mark.skip(reason="not implemented")
+ def test_nb_sequences(self, calibrate_call: CalibrateCall):
+ notebook_path = (
+ calibrate_call.paths.notebooks[0].parent
+ / "AGIPD_Correct_and_Verify__sequences__1.ipynb"
+ )
+
+ with notebook_path.open() as file:
+ notebook = nbformat.read(file, as_version=4)
+
+ parameters = {p.name: p.value for p in extract_parameters(notebook)}
+ # TODO: add test cases for this notebook
+ print(parameters)
+
+ @pytest.mark.skip(reason="not implemented")
+ def test_nb_summary(self, calibrate_call: CalibrateCall):
+ notebook_path = (
+ calibrate_call.paths.notebooks[0].parent
+ / "AGIPD_Correct_and_Verify_Summary_NBC__None__None.ipynb"
+ )
+
+ with notebook_path.open() as file:
+ notebook = nbformat.read(file, as_version=4)
+
+ parameters = {p.name: p.value for p in extract_parameters(notebook)}
+ # TODO: add test cases for this notebook
+ print(parameters)
+
+ @pytest.mark.skip(reason="not implemented")
+ def test_nb_precorrection(self, calibrate_call: CalibrateCall):
+ notebook_path = (
+ calibrate_call.paths.notebooks[0].parent
+ / "AGIPD_Retrieve_Constants_Precorrection__None__None.ipynb"
+ )
+
+ with notebook_path.open() as file:
+ notebook = nbformat.read(file, as_version=4)
+ # TODO: add test cases for this notebook
+ parameters = {p.name: p.value for p in extract_parameters(notebook)}
+
+ print(parameters)
diff --git a/webservice/webservice.py b/webservice/webservice.py
index ec97eeb24fe4cfe9bd160f92640666bd1cf47a20..494608fd7223efa1ce10ce873d3eeb0e03af3476 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -1087,11 +1087,18 @@ class ActionsServer:
request_time
)
await update_mdc_status(self.mdc, 'dark_request', rid, ret)
- if report_path is None:
+ if len(report_path) == 0:
logging.warning("Failed to identify report path for dark_request")
else:
- await update_darks_paths(self.mdc, rid, in_folder,
- out_folder, report_path)
+ if len(report_path) > 1:
+ logging.warning(
+ "More than one report path is returned. "
+ "Updating myMDC with the first report path only."
+ )
+ await update_darks_paths(
+ self.mdc, rid, in_folder,
+ out_folder, report_path[0]
+ )
# END of part to run after sending reply
asyncio.ensure_future(_continue())
@@ -1162,7 +1169,9 @@ class ActionsServer:
async def launch_jobs(
self, run_nrs, rid, detectors, action, instrument, cycle, proposal,
request_time
- ) -> (str, Optional[str]):
+ ) -> (str, List[str]):
+ report = []
+ ret = []
# run xfel_calibrate
for karabo_id, dconfig in detectors.items():
detector = dconfig['detector-type']
@@ -1179,16 +1188,16 @@ class ActionsServer:
).split()
cmd = parse_config(cmd, dconfig)
-
- ret = await run_action(self.job_db, cmd, self.mode,
- proposal, run_nrs[-1], rid)
-
+ # TODO: Add detector info in returned run action status.
+ ret.append(await run_action(
+ self.job_db, cmd, self.mode,
+ proposal, run_nrs[-1], rid
+ ))
if '--report-to' in cmd[:-1]:
report_idx = cmd.index('--report-to') + 1
- report = cmd[report_idx] + '.pdf'
- else:
- report = None
- return ret, report
+ report.append(cmd[report_idx] + '.pdf')
+ # return string without a tailing comma.
+ return ", ".join(ret), report
parser = argparse.ArgumentParser(
description='Start the calibration webservice')