diff --git a/.gitignore b/.gitignore
index 7e45073ed70843fde1ea1914457fdc44ccbff222..7f2b6feb4f30aa8d9281b22cf2aab62b154f937e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,6 +32,7 @@ docs/source/test_results.rst
docs/source/test_rsts
reportservice/*.log
slurm_tmp*
-src/cal_tools/agipdalgs.c
+src/cal_tools/*.c
+src/cal_tools/*/*.c
webservice/*.log
webservice/*sqlite
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bb8dd05a06ad3919adcada7e255575965ade5abe..c4d3d5ba6246b038763fc27a2199497d8682d109 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -57,4 +57,4 @@ cython-editable-install-test:
<<: *before_script
script:
- python3 -m pip install -e ".[test]"
- - python3 -m pytest --color yes --verbose ./tests/test_agipdalgs.py
+ - python3 -m pytest --color yes --verbose ./tests/test_cythonalgs.py
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 8c3d97346d10554acfffb13cc0867b8609e11852..b3c28f3e8f7a208d68e3583b9fad97982a28037f 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1 +1 @@
-iCalibrationDB @ git+https://xcalgitlab:${GITHUB_TOKEN}@git.xfel.eu/gitlab/detectors/cal_db_interactive.git@2.0.9
\ No newline at end of file
+iCalibrationDB @ git+https://xcalgitlab:${GITHUB_TOKEN}@git.xfel.eu/gitlab/detectors/cal_db_interactive.git@2.2.0
diff --git a/docs/source/cal_tools_algorithms.rst b/docs/source/cal_tools_algorithms.rst
new file mode 100644
index 0000000000000000000000000000000000000000..3698e0ede66a9776d39f2498f1281df3781501ee
--- /dev/null
+++ b/docs/source/cal_tools_algorithms.rst
@@ -0,0 +1,22 @@
+cal_tools
+=========
+
+.. module:: cal_tools.agipdlib
+
+.. class:: AgipdCorrections
+
+ .. attribute:: read_file
+
+ .. attribute:: write_file
+
+ .. attribute:: cm_correction
+
+ .. attribute:: mask_zero_std
+
+ .. attribute:: offset_correction
+
+ .. attribute:: baseline_correction
+
+ .. attribute:: gain_correction
+
+ .. attribute:: get_valid_image_idx
\ No newline at end of file
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
new file mode 100644
index 0000000000000000000000000000000000000000..f2f266a62756c6a5161049ac4a07046d17087d29
--- /dev/null
+++ b/docs/source/changelog.rst
@@ -0,0 +1,193 @@
+Release Notes
+=============
+
+3.5.5
+-----
+
+15-06-2022
+
+- [AGIPD][CORRECT] Expose max tasks per pool worker.
+
+3.5.4
+-----
+
+13-06-2022
+
+- [AGIPD] Convert bias_voltage parameter condition to integer in cal_tools.
+- [LPD] Fix correcting a single pulse.
+- [LPD] VCXI require 4 modules.
+
+3.5.3
+-----
+
+19-05-2022
+
+- [LPD][CORRECT] Optionally create virtual CXI files
+- [LPD][CORRECT] Expose max-nodes parameter
+- [AGIPD] Replace gain_choose_int by fused types
+- Fix missing install of restful_config.yaml
+- Fix use of xfel-calibrate --skip-report
+
+3.5.2
+-----
+
+16.05.2022
+
+- [LPD][CORRECT] New correction notebook for LPD
+- New `files` module to write European XFEL HDF5 corrected data files.
+
+3.5.1
+-----
+
+05-04-2022
+
+- Calibration Constant version's new `Variant` file attribute. To indicate method of handling the constant post retrieval. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/619
+- Epix100 dark Badpixels Map. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/637
+- `skip-plots` flag to finish correction before plotting. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/635
+- First trainId's timestamp as RAW data creation_time, if there is myMDC connection. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/630
+- AGIPD correction can correct one cellId without plotting errors. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/642
+- Fixed mode relative gain constants in Jungfrau can be retrieved. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/639
+- Only instrument source is selected to check number of trains to dark process. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/636
+- AGIPD trains for dark processing is selected for each module individually. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/653
+- Produce report after trying to correct AGIPD run with no images. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/641
+- AGIPD's bias voltage for AGIPD1M is read from slow data. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/647
+- Removed psutil dependency. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/653
+- Update Pasha to 0.1.1 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/638
+
+3.5.0
+-----
+
+01-03-2022
+
+- Updating Correction and dark notebooks for JUNGFRAU: https://git.xfel.eu/detectors/pycalibration/-/merge_requests/518
+- Updating Correction and dark notebooks for AGIPD: https://git.xfel.eu/detectors/pycalibration/-/merge_requests/535
+- Updating Correction and dark notebooks for PnCCD: https://git.xfel.eu/detectors/pycalibration/-/merge_requests/559
+- Updating Correction and dark notebooks for ePix100: https://git.xfel.eu/detectors/pycalibration/-/merge_requests/500
+
+ * EXtra-data is integrated to read files in pycalibration for AGIPD, JUNGFRAU, ePix100, and PnCCD. Dark and Correction notebooks.
+ * Pasha is now used for processing data for JUNGFRAU, ePix100 and PnCCD.
+ * pyDetLib correction functions were removed (except for common-mode correction).
+ * `db-module` is useless now for JUNGFRAU, ePix100 and PnCCD. Some parameters were updated in dark and correction notebooks for the mentioned detectors.
+
+- `gain_mode` and burst mode are now available for JUNGFRAU. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/591
+- JUNGFRAU has now a new badpixel value, `WRONG_GAIN_VALUE`. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/591
+- Pass through available for testing in-progress ORCA service. https://git.xfel.eu/detectors/pycalibration/-/merge_requests?scope=all&state=merged&search=orca
+- Non-calibrated RAW h5files are no longer copied.
+- High priority partitions (`upex-high`and `upex-middle`) are used for runs from ACTIVE and READY proposals, only. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/628
+- Supporting to disable LPD Correction through the webservice. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/629
+- Compatibility for old DAQ files for REMI is added. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/607
+- server-overview refactors. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/593 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/589
+- AGIPD correction notebook support AgipdLitFrameFinder device. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/603
+- Parsing code arguments in xfel-calibrate is refactored. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/575
+- skip-plots option for AGIPD. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/581
+- Native implementation for transposition of constants AGIPD. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/580
+- Trains for AGIPD can be selected for correction. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/574
+- Skip report flag in xfel-calibrate. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/579
+- Fix ReadTheDocs. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/448
+- Fix error reporting for re-injecting the same CCV. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/627
+- Fix AGIPD for legacy runs without `gain_mode`. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/617 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/626
+- Pinning markupsafe version 2.0.1 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/631
+- Pinning psutil 5.9.0 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/535
+- Updating Extra-data to 1.9.1 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/535
+- Updating h5py to 3.5.0 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/602
+
+3.4.3
+-----
+
+20-10-2021
+
+- Update pyDetLib tag.
+- Add explicit dependencies on matplotlib, scipy.
+- Remove outdated matplotlib rcParams setting.
+- Update EXtra-geom to 1.6.
+- Remove cluster_profile parameter from notebooks which don't use it.
+- Fix checking availability for the concurrency parameter.
+- Fix launching work directly (not via Slurm).
+- Fix `sphinx-rep` temp folder recreation, if sphinx-rep already existed.
+- Fix missing string conversion for slurm-scheduling argument.
+- Fix title reports for multiple detectors per run folder.
+- Append to .out files for preemptable finalize job.
+- [AGIPD] [Correct] Reuse previously found constants.
+- [AGIPD] Fix missing memory cell index in SlopesPC constant sanitization.
+- [AGIPD] Only use bad pixels from darks in agipdutils.baseline_correct_via_stripes.
+- [AGIPD] [Dark] Use function to get list of karabo_da from run for making Slurm jobs.
+- [EPIX100][CORRECT] Set absolute_gain to false if relative gain was not retrieved.
+- [JUNGFRAU] Fix running for multiple modules and flip logic for do_relative_gain.
+- [JUNGFRAU] Style changes for Dark and Correct notebooks.
+- [REMI] Add notebook to reconstruct detector hits from raw data.
+- [webservice] Check run migration status using MyMDC.
+- Resolve "Skip ZMQ tests if zmq connection for calibration DB not available".
+- Reproducibility, step 1.
+
+3.4.2
+-----
+
+17-09-2021
+
+- Remove driver=core from all notebook
+- [webservice] Make use of Dynaconf for managing secrets.
+- [webservice] Make use of dedicated slurm partitions.
+- [webservice] Handle missing migration information (missing user.status fattr).
+- [webservice] Implement, raise, and catch, migration errors to send mdc messages.
+- [webservice] Simplify handling of user notebook paths.
+- [webservice] Update princess to 0.4 (use Unix sockets).
+- [webservice] Update MyMDC with begin and end times.
+- [webservice] create output folder before copying slow data.
+- [AGIPD] [CORRECT] read acq_rate from slow data.
+- [AGIPD][CORRECT] Set default memory cells to 352.
+- [AGIPD] [CORRECT] Set maximum pulses to correct based on file content.
+- [AGIPD] [FF] Correctly label legends in figures.
+- [AGIPD] [FF] Add HIBEF AGIPD500K and fix some issue with retrieval of conditions.
+- [Jungfrau] Add Gain setting to Jungfrau notebooks.
+- [Jungfrau] Fix max gain plot in LPD correct notebook
+- [JUNGFRAU] [DARK] Clearer error message for Jungfrau Dark notebooks no suitable files are found
+- [LPD] [CORRECT] Fix max gain plot.
+- [EPIX100] [CORRECT] Solve conflict between gain correction and clustering
+
+
+3.4.1
+-----
+
+16-07-2021
+
+- Update h5py to 3.3
+- Stop execution on notebook errors
+- [AGIPD] Add integration time as operating condition to all notebooks
+- [webservice] Add blocklist pattern when copying untouched files in webservice.
+- [webservice] Expose dark configurations in update_config.py
+- Fix MetadataClient.get_proposal_runs arguments call.
+- Fix Use snapshot for injecting constants for old PDU mappings
+- Fix the old time-summary (creation time for retrieved constants)
+- Update documentation notes on venv installation
+- Ignore all .so files in gitignore
+
+
+3.4.0
+-----
+
+28-06-2021
+
+- Update to Python 3.8.
+- Bump numpy to 1.20.3 and remove fabio.
+- remove PyQT dependency.
+- Disable dark requests from serve overview.
+- Update report upload parameter key.
+- Override locale to always use UTF-8.
+- Assorted cleanup of xfel-calibrate.
+- Fix pre-commit.
+- Use argparse only if name is main, call main with args dict.
+- [webservice] Use full hostname for webservice overview.
+- [webservice] Show clearer messages when running webservice in sim mode.
+- [webservice] Fix filename lineno and typos in webservice logs.
+- [webservice] Fix creating an extra run folder in run output folder.
+- [AGIPD] Parallelize gain/mask compression for writing corrected AGIPD files.
+- [AGIPD][DARK] Fix processing empty sequence files.
+- [AGIPD][PC][FF] Update notebooks with new CALCAT mapping.
+- [AGIPD][JUNGFRAU] Use all available sequences for processing darks for AGIPD and Jungfrau.
+- [AGIPD][LPD][DSSC] Fix retrieve old constants for comparison for modular detectors.
+- [LPD] Fix data paths in LPD notebook.
+- [REMI] Fix user notebook path for REMI correct notebook provisionally.
+- [EPIX][CORRECT] Add Common mode correction.
+- Fix plotting-related warnings.
+- Test update config.
+- Test get_from_db and send_to_db.
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 5112b0010d5a26c2fbe9d20ac57d4604006997dc..e4827618883ef2cf834e57a209eaf083ceb2b1eb 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -150,7 +150,7 @@ todo_include_todos = True
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
-# html_theme = ''
+html_theme = 'sphinx_rtd_theme'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
@@ -403,6 +403,9 @@ with open("available_notebooks.rst", "w") as f:
"""))
for detector in sorted(notebooks.notebooks.keys()):
+ # Avoid having test notebooks in detector notebooks documentations.
+ if "TEST" in detector.upper():
+ continue
values = notebooks.notebooks[detector]
f.write("{}\n".format(detector))
f.write("{}\n".format("-"*len(detector)))
@@ -410,8 +413,6 @@ with open("available_notebooks.rst", "w") as f:
for caltype in sorted(values.keys()):
data = values[caltype]
- if data.get("notebook", None) is None:
- continue
nbpath = os.path.abspath("{}/../../../{}".format(__file__, data["notebook"]))
with open(nbpath, "r") as nf:
nb = nbformat.read(nf, as_version=4)
@@ -442,7 +443,6 @@ with open("available_notebooks.rst", "w") as f:
output = check_output(nb_help).decode('utf8')
f.write(indent(output.replace("DETECTOR", detector).replace("TYPE", caltype), " "*4))
f.write("\n\n")
-
# add test results
test_artefact_dir = os.path.realpath("../../tests/legacy/artefacts")
diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst
index 241070cbd51c9e0eab655e9f8920c5878c1db17c..750cdb5475e0f45247ca863e3efe51abd40ae805 100644
--- a/docs/source/configuration.rst
+++ b/docs/source/configuration.rst
@@ -3,16 +3,14 @@
Configuration
=============
-The European XFEL Offline Calibration tools are configure using the `settings.py`
-and `notebooks.py` files. Both can be found in the root directory. The `settings.py`
-file configures the tools to the environment. The `notebook.py` file configures the
-notebooks which should be exposed to the command line.
+The European XFEL Offline Calibration is configured through `settings.py`
+and `notebooks.py` files. Both can be found in `xfel_calibrate` source directory. The `notebook.py` file exposes and configures the
+the notebooks by the `xfel-calibrate` command line interface.
Settings
--------
-The `settings.py` file configures the enviroment the tools are run in. It is a normal
-python file of the form::
+The `settings.py` is a python configuration file, which configures the tool's environment.::
# path into which temporary files from each run are placed
temp_path = "{}/temp/".format(os.getcwd())
@@ -35,9 +33,9 @@ A comment is given for the meaning of each configuration parameter.
Notebooks
---------
-The `xfel-calibrate` tool will expose any notebooks that are configured here to the
+The `xfel-calibrate` tool exposes configured notebooks to the
command line by automatically parsing the parameters given in the notebooks first cell.
-The configuration is to be given in form of a python directory::
+The configuration is given in the form of a python dictionary::
notebooks = {
"AGIPD": {
@@ -63,55 +61,23 @@ The configuration is to be given in form of a python directory::
}
}
-The first key is the detector that the calibration may be used for, here AGIPD. The second
-key level gives the name of the task being performed (here: DARK and PC). For each of these
-entries, a path to the notebook and a concurrency hint should be given. In the concurrency
-hint the first entry specifies which parameter of the notebook expects a list whose integer
-entries, can be concurrently run (here "modules"). The second parameter state with which range
-to fill this parameter if it is not given by the user. In the example a `range(16):=0,1,2,...15`
-would be passed onto the notebook, which is run as 16 concurrent jobs, each processing one module.
-Finally, a hint for the number of cluster cores to be started should be given. This value should
-be derived e.g. by profiling memory usage per core, run times, etc.
+The first key is the detector, e.g. AGIPD. The second key is the calibration type name, e.g. DARK or PC.
+A dictionary is expected for each calibration type with a notebook path and concurrency configuration.
+For the concurrency three values are expected. Key `parameter` with a value name of type list, which is defined in the first notebook cell.
+The key `default concurrency` to define the range of values for `parameter` in each concurrent notebook, if it is not defined by the user.
+e.g. `"default concurrency": 16` leads to running 16 concurrent jobs, each processing one module with values of [0,1,2,...,15].
+Finally, a hint for the number of `cluster cores` that is used if the notebook is using ipcluster parallelization, only.
+This value should be derived e.g. by profiling memory usage per core, run times, etc.
.. note::
It is good practice to name command line enabled notebooks with an `_NBC` suffix as shown in
the above example.
-The `CORRECT` notebook (last notebook in the example) makes use of a concurrency generating function
-by setting the `use function` parameter. This function must be defined in a code cell in the notebook,
-its parameters should be named like other exposed parameters. It should return a list of of parameters
-to be inserted into the concurrently run notebooks. The example given e.g. defines the `balance_sequences`
-function::
-
- def balance_sequences(in_folder, run, sequences, sequences_per_node):
- import glob
- import re
- import numpy as np
- if sequences_per_node != 0:
- sequence_files = glob.glob("{}/r{:04d}/*-S*.h5".format(in_folder, run))
- seq_nums = set()
- for sf in sequence_files:
- seqnum = re.findall(r".*-S([0-9]*).h5", sf)[0]
- seq_nums.add(int(seqnum))
- seq_nums -= set(sequences)
- return [l.tolist() for l in np.array_split(list(seq_nums),
- len(seq_nums)//sequences_per_node+1)]
- else:
- return sequences
-
-
-.. note::
-
- Note how imports are inlined in the definition. This is necessary, as only the function code,
- not the entire notebook is executed.
-
-which requires as exposed parameters e.g. ::
-
- in_folder = "/gpfs/exfel/exp/SPB/201701/p002038/raw/" # the folder to read data from, required
- run = 239 # runs to process, required
- sequences = [-1] # sequences to correct, set to -1 for all, range allowed
- sequences_per_node = 2 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel
+The AGIPD `CORRECT` notebook (last notebook in the example) makes use of a concurrency generating function
+by setting the `use function` parameter. This function must be defined in the first cell in the notebook
+its given arguments should be named as the first cell notebook parameters. It is expected to return a list of parameters
+to concurrently run notebooks. Above the used function is :func:`xfel_calibrate.calibrate.balance_sequences`.
.. note::
diff --git a/docs/source/how_it_works.rst b/docs/source/how_it_works.rst
deleted file mode 100644
index 090ebae9fc97591f86af5b71cac651b555faa63d..0000000000000000000000000000000000000000
--- a/docs/source/how_it_works.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-.. _how_it_works:
-
-How it Works
-============
-
-The European XFEL Offline Calibration utilizes the tools nbconvert_ and nbparameterise_
-to expose Jupyter_ notebooks to a command line interface. In the process reports are generated
-from these notebooks. The general interface is::
-
- % xfel-calibrate DETECTOR TYPE
-
-where `DETECTOR` and `TYPE` specify the task to be performed.
-
-Additionally, it leverages the DESY/XFEL Maxwell cluster to run these jobs in parallel
-via SLURM_.
-
-Here is a list of :ref:`available_notebooks`. See the :ref:`advanced_topics` if you are
-for details on how to use as detector group staff.
-
-If you would like to integrate additional notebooks please see the :ref:`development_workflow`.
-
-.. _nbparameterise: https://github.com/takluyver/nbparameterise
-.. _nbconver: https://github.com/jupyter/nbconvert
-.. _jupyter: http://jupyter.org/
-.. _SLURM: https://slurm.schedmd.com
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 5512e8a006aec326d32864ed11b2321248d428f3..b57a68dee27860df4384878f2116ba92a7c4740f 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -3,15 +3,45 @@
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
-Welcome to European XFEL Offline Calibration's documentation!
-=============================================================
+European XFEL Offline Calibration
+=================================
-Contents:
+The European XFEL Offline Calibration (pyCalibration) is a python package that consists of
+different services, responsible for applying most of the offline calibration
+and characterization for the detectors.
+
+Running a calibration
+---------------------
+
+It utilizes tools such as nbconvert_ and nbparameterise_
+to expose Jupyter_ notebooks to a command line interface.
+In the process reports are generated from these notebooks.
+
+The general interface is::
+
+ % xfel-calibrate DETECTOR TYPE
+
+where `DETECTOR` and `TYPE` specify the task to be performed.
+
+Additionally, it leverages the DESY/XFEL Maxwell cluster to run these jobs in parallel
+via SLURM_.
+
+Here is a list of :ref:`available_notebooks`. See the :ref:`advanced_topics` if you are looking
+for details on how to use as detector group staff.
+
+If you would like to integrate additional notebooks please see the :ref:`development_workflow`.
+
+.. _nbparameterise: https://github.com/takluyver/nbparameterise
+.. _nbconvert: https://github.com/jupyter/nbconvert
+.. _jupyter: http://jupyter.org/
+.. _SLURM: https://slurm.schedmd.com
+
+
+Documentation contents:
.. toctree::
:maxdepth: 2
- how_it_works
installation
configuration
workflow
@@ -20,3 +50,23 @@ Contents:
tutorial
_notebooks/index
testing
+
+.. toctree::
+ :caption: Reference
+ :maxdepth: 2
+
+ xfel_calibrate_conf
+ cal_tools_algorithms
+
+.. toctree::
+ :caption: Development
+ :maxdepth: 2
+
+ changelog
+ architecture
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`search`
\ No newline at end of file
diff --git a/docs/source/installation.rst b/docs/source/installation.rst
index 58f3f72268b460d4adaf297c0e6ede83ba2a0a1d..97d83ad8c40fa9c4b84a30125a671689173ab6e3 100644
--- a/docs/source/installation.rst
+++ b/docs/source/installation.rst
@@ -1,118 +1,100 @@
-Installation
-============
-
-Installation of the European XFEL Offline Calibration tools is best
-done into an anaconda/3 environment using Maxwell_ cluster. However, there are no explicit
-dependencies, so any Python3 environment is capable of running the tool.
-
-A more detailed step-by-step instruction can be found in the :ref:`tutorial`.
-
-Package Requirements
---------------------
-
-Requirements can be categorized into three types:
-
-1. Those required to run the the general tool chain,
-2. Those required by the notebooks themselves,
-3. Those required to run notebooks concurrently on a HPC cluster.
-
-Categories 1+2 are usually Python modules, as given in the `requirements.txt`
-file found in the root directory. Additionally, PyDetLib_ is required.
-It can be installed either as a Karabo dependency or manually.
-
-Parallel execution is currently supported in a `SLURM_` environment.
-However, sequential execution is available using the `--no-cluster-job`
-parameter when executing `xfel-calibrate`.
-
-.. _Python: http://www.python.org/
-.. _Karabo: https://in.xfel.eu/readthedocs/docs/karabo/en/latest/
-.. _SLURM: https://slurm.schedmd.com
-.. _PyDetLib: https://in.xfel.eu/readthedocs/docs/pydetlib/en/latest/
-.. _Maxwell: https://confluence.desy.de/display/IS/Running+Jobs+on+Maxwell
-
-Installation using Anaconda
----------------------------
-
-First you need to load the anaconda/3 environment through::
+.. _installation:
- 1. module load anaconda/3
-
-If installing into other python enviroments, this step can be skipped.
-
-Then the package for the offline calibration can be obtained from the git repository::
-
- 2. git clone https://git.xfel.eu/gitlab/detectors/pycalibration.git
-
-Home directory
-++++++++++++++
-
-You can then install all requirements of this tool chain in your home directory by running::
-
- 3. pip install -r requirements.txt . --user
-
-in pycalibration's root directory.
-
-After installation, you should make sure that the home directory is in the PATH environment variable::
-
- 4. PATH=/home/<username>/.local/bin:$PATH
+************
+Installation
+************
-Preferred directory
-+++++++++++++++++++
+It's recommended to install the offline calibration (pycalibration) package on
+maxwell, using the anaconda/3 environment.
-Alternatively, you can install all requirements in a directory of your preference by::
+The following instructions clone from the EuXFEL GitLab instance using SSH
+remote URLs, this assumes that you have set up SSH keys for use with GitLab
+already. If you have not then read the appendix section on `SSH Key Setup for
+GitLab`_ for instructions on how to do this .
- 3. mkdir /gpfs/exfel/data/scratch/<username>/<directory-name>
- pip install --target=/gpfs/exfel/data/scratch/<username>/<directory-name> -r requirements.txt .
-and it is important to make sure that the installed requirements are in the PATH environment::
+Installation using python virtual environment - recommended
+===========================================================
- 4. PATH=/gpfs/exfel/data/scratch/<username>/<directory-name>/bin:$PATH
+`pycalibration` uses the same version of Python as Karabo, which in June 2021
+updated to use Python 3.8. Currently the default python installation on Maxwell
+is still Python 3.6.8, so Python 3.8 needs to be loaded from a different
+location.
+One option is to use the Maxwell Spack installation, running `module load
+maxwell` will activate the test Spack instance from DESY, then you can use
+`module load python-3.8.6-gcc-10.2.0-622qtxd` to Python 3.8. Note that this Spack
+instance is currently a trial phase and may not be stable.
-After this make sure that you adjust the :ref:`configuration` and settings in the xfel-calibrate
-folder to match your environment.
+Another option is to use `pyenv`, we provide a pyenv installation at
+`/gpfs/exfel/sw/calsoft/.pyenv` which we use to manage different versions of
+python. This can be activated with ``source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate``
-The tool-chain is then available via the::
+A quick setup would be:
- xfel-calibrate
+1. ``source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate``
+2. ``git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git && cd pycalibration`` - clone the offline calibration package from EuXFEL GitLab
+3. ``pyenv shell 3.8.11`` - load required version of python
+4. ``python3 -m venv .venv`` - create the virtual environment
+5. ``source .venv/bin/activate`` - activate the virtual environment
+6. ``python3 -m pip install --upgrade pip`` - upgrade version of pip
+7. ``python3 -m pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
-command.
+Copy/paste script:
+.. code:: bash
-Installation using karabo
-+++++++++++++++++++++++++
+ source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate
+ git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git
+ cd pycalibration
+ pyenv shell 3.8.11
+ python3 -m venv .venv
+ source .venv/bin/activate
+ python3 -m pip install --upgrade pip
+ python3 -m pip install . # `-e` flag for editable install, e.g. `pip install -e .`
-If required, one can install into karabo environment. The difference would be to
-first source activate the karabo envrionment::
- 1. source karabo/activate
+Creating an ipython kernel for virtual environments
+===================================================
-then after cloning the offline calibration package from git, the requirements can be installed through::
+To create an ipython kernel with pycalibration available you should (if using a
+venv) activate the virtual environment first, and then run:
- 3. pip install -r requirements.txt .
+.. code:: bash
-Development Installation
-------------------------
+ python3 -m pip install ipykernel # If not using a venv add `--user` flag
+ python3 -m ipykernel install --user --name pycalibration --display-name "pycalibration" # If not using a venv pick different name
-For a development installation in your home directory, which automatically
-picks up (most) changes, first install the dependencies as above,
-but then install the tool-chain separately in development mode::
+This can be useful for Jupyter notebook tools as https://max-jhub.desy.de/hub/login
- pip install -e . --user
-.. note:: Using "- -target" for development installation in a preferred directory can lead to errors.
+SSH Key Setup for GitLab
+========================
-.. note:: For development installation in karabo environment "- -user" is not needed.
+It is highly recommended to set up SSH keys for access to GitLab as this
+simplifies the setup process for all of our internal software present on GitLab.
-Installation of New Notebooks
------------------------------
+To set up the keys:
-To install new, previously untracked notebooks in the home directory,
-repeat the installation of the the tool-chain, without requirments,
-from the package base directory::
+1. Connect to Maxwell
+2. Generate a new keypair with ``ssh-keygen -o -a 100 -t ed25519``, you can
+ either leave this in the default location (``~/.ssh/id_ed25519``) or place it
+ into a separate directory to make management of keys easier if you already
+ have multiple ones. If you are using a password for your keys please check
+ this page to learn how to manage them: https://docs.github.com/en/github/authenticating-to-github/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent#adding-your-ssh-key-to-the-ssh-agent
+3. Add the public key (``id_ed25519.pub``) to your account on GitLab: https://git.xfel.eu/gitlab/profile/keys
+4. Add the following to your ``~/.ssh/config`` file
- pip install --upgrade . --user
+.. code::
-Or, in case you are actively developing::
+ # Special flags for gitlab over SSH
+ Host git.xfel.eu
+ User git
+ Port 10022
+ ForwardX11 no
+ IdentityFile ~/.ssh/id_ed25519
- pip install -e . --user
+Once this is done you can clone repositories you have access to from GitLab
+without having to enter your password each time. As ``pycalibration``
+requirements are installed from SSH remote URLs having SSH keys set up is a
+requirement for installing pycalibration.
diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst
index 609d97bf6119c8ef034eede79a943f7a2a88202e..450b00464705cdd887f27264bcd3da05a533cadd 100644
--- a/docs/source/tutorial.rst
+++ b/docs/source/tutorial.rst
@@ -25,41 +25,7 @@ This will open a jupyter kernel running in your browser where you can then open
ipcluster start --n=4 --profile=tutorial
you can step through the cells and run them.
-If you run this notebook using the xfel-calibrate command as explaind at the end of this tutorial you do not need to start the cluster yourself, it will be done by the framework.
-
-
-Installation and Configuration
-------------------------------
-
-The offline calibration tool-chain is optimised to run on the maxwell cluster.
-For more information refer to the Maxwell_ documentation.
-
-.. _Maxwell: https://confluence.desy.de/display/IS/Running+Jobs+on+Maxwell
-
-In order to use the offline calibration tool a few steps need to be carried out
-to install the necessary packages and setup the environment:
-
-1. Log into max-exfl with you own user name/account.
-
-2. Install karabo in your home directory or under /gpfs/exfel/data/scratch/username
- by typing the following commands on you shell::
-
- wget http://exflserv05.desy.de/karabo/karaboFramework/tags/2.2.4/karabo-2.2.4-Release-CentOS-7-x86_64.sh
-
- chmod +x karabo-2.2.4-Release-CentOS-7-x86_64.sh
-
- ./karabo-2.2.4-Release-CentOS-7-x86_64.sh
-
- source karabo/activate
-
-3. Get the package pycalibration which contains the offline calibration tool-chain::
-
- git clone https://git.xfel.eu/gitlab/detectors/pycalibration.git
-
-4. Install the necessary requirements and the package itself::
-
- cd pycalibration
- pip install -r requirements.txt .
+If you run this notebook using the xfel-calibrate command as explained at the end of this tutorial you do not need to start the cluster yourself, it will be done by the framework.
Create your own notebook
@@ -87,7 +53,7 @@ Running the notebook
You can see your job in the queue with::
- squeue -u username
+ squeue --me
3. Look at the generated report in the chosen output folder.
4. More information on the job run on the cluster can be found in the temp folder.
diff --git a/docs/source/workflow.rst b/docs/source/workflow.rst
index 0b9f0c6ff2d8870b18b0f85959c62d548f1c0ca5..95ab78463ee362a3bdcba2f0d31067b69e143a48 100644
--- a/docs/source/workflow.rst
+++ b/docs/source/workflow.rst
@@ -4,7 +4,7 @@ Development Workflow
====================
The following walkthrough will guide you through a possible workflow
-when developing new offline calibration tools.
+when developing new notebooks for offline calibration.
Fresh Start
-----------
@@ -12,7 +12,7 @@ Fresh Start
If you are starting a blank notebook from scratch you should first
think about a few preconsiderations:
-* Will the notebook performan a headless task, or will it also be
+* Will the notebook perform a headless task, or will it also be
an important interface for evaluating the results in form of a
report.
* Do you need to run concurrently? Is concurrency handled internally,
@@ -25,7 +25,7 @@ cells in the notebook. You should also structure it into appropriate
subsections.
If you plan on running concurrently on the cluster, identify which variable
-should be mapped to concurent runs. For autofilling it an integer list is
+should be mapped to concurrent runs. For autofilling it an integer list is
needed.
Once you've clarified the above points, you should create a new notebook,
@@ -139,7 +139,7 @@ to the following parameters being exposed via the command line::
.. note::
- Nbparameterise can only parse the mentioned subset of variable types. An expression
+ nbparameterise_ can only parse the mentioned subset of variable types. An expression
that evaluates to such a type will note be recognized: e.g. `a = list(range(3))` will
not work!
@@ -170,59 +170,41 @@ Best Coding Practices
In principle there a not restrictions other than that parameters that are exposed to the
command line need to be defined in the first code cell of the notebook.
-However, a few guidelines should be observered to make notebook useful for display as
-reports and usage by other.
+However, a few guidelines should be observed to make notebook useful for display as
+reports and usage by others.
External Libraries
~~~~~~~~~~~~~~~~~~
-You may use a wide variaty of libraries available in Python, but keep in mind that others
+You may use a wide variety of libraries available in Python, but keep in mind that others
wanting to run the tool will need to install these requirements as well. Thus,
-* do not use a specialized tool if an accepted alternative exists. Plots e.g. should usually
+* Do not use a specialized tool if an accepted alternative exists. Plots e.g. should usually
be created using matplotlib_ and numerical processing should be done in numpy_.
-* keep runtimes and library requirements in mind. A library doing its own parallelism either
- needs to programatically be able to set this up, or automatically do so. If you need to
+* Keep runtime and library requirements in mind. A library doing its own parallelism either
+ needs to programmatically be able to set this up, or automatically do so. If you need to
start something from the command line first, things might be tricky as you will likely
need to run this via `POpen` commands with appropriate environment variable.
+* Reading out RAW data should be done using extra_data_. It helps in accessing the HDF5 data
+ structures efficiently. It reduces the complexity of accessing the RAW or CORRECTED datasets,
+ and it provides different methods to select and filter the trains, cells, or pixels of interest.
+
Writing out data
~~~~~~~~~~~~~~~~
If your notebook produces output data, consider writing data out as early as possible,
such that it is available as soon as possible. Detailed plotting and inspection can
-possibly done later on in a notebook.
+be done later on in the notebook.
-Also consider using HDF5 via h5py_ as your output format. If you correct or calibrated
-input data, which adhears to the XFEL naming convention, you should maintain the convention
+Also use HDF5 via h5py_ as your output format. If you correct or calibrate
+input data, which adheres to the XFEL naming convention, you should maintain the convention
in your output data. You should not touch any data that you do not actively work on and
-should assure that the `INDEX` and identifier entries are syncronized with respect to
+should assure that the `INDEX` and identifier entries are synchronized with respect to
your output data. E.g. if you remove pulses from a train, the `INDEX/.../count` section
should reflect this.
-Finally, XFEL RAW data can contain filler data from the DAQ. One possible way of identifying
-this data is the following::
-
- datapath = "/INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/cellId".format(channel)
-
- count = np.squeeze(infile[datapath])
- first = np.squeeze(infile[datapath])
- if np.count_nonzero(count != 0) == 0: # filler data has counts of 0
- print("File {} has no valid counts".format(infile))
- return
- valid = count != 0
- idxtrains = np.squeeze(infile["/INDEX/trainId"])
- medianTrain = np.nanmedian(idxtrains) # protect against freak train ids
- valid &= (idxtrains > medianTrain - 1e4) & (idxtrains < medianTrain + 1e4)
-
- # index ranges in which non-filler data exists
- last_index = int(first[valid][-1]+count[valid][-1])
- first_index = int(first[valid][0])
-
- # access these indices
- cellIds = np.squeeze(np.array(infile[datapath][first_index:last_index, ...]))
-
Plotting
~~~~~~~~
@@ -233,10 +215,10 @@ a context. Make sure to label your axes.
Also make sure the plots are readable on an A4-sized PDF page; this is the format the notebook
will be rendered to for report outputs. Specifically, this means that figure sizes should not
-exeed approx 15x15 inches.
+exceed approx 15x15 inches.
The report will contain 150 dpi png images of your plots. If you need higher quality output
-of individual plot files you should save these separetly, e.g. via `fig.savefig(...)` yourself.
+of individual plot files you should save these separately, e.g. via `fig.savefig(...)` yourself.
Calibration Database Interaction
@@ -245,7 +227,7 @@ Calibration Database Interaction
Tasks which require calibration constants or produce such should do this by interacting with
the European XFEL calibration database.
-In terms of developement workflow it is usually easier to work with file-based I/O first and
+In terms of development workflow it is usually easier to work with file-based I/O first and
only switch over to the database after the algorithmic part of the notebook has matured.
Reasons for this include:
@@ -261,7 +243,7 @@ documentation.
Testing
-------
-The most important test is that your notebook completes flawlessy outside any special
+The most important test is that your notebook completes flawlessly outside any special
tool chain feature. After all, the tool chain will only replace parameters, and then
launch a concurrent job and generate a report out of notebook. If it fails to run in the
normal Jupyter notebook environment, it will certainly fail in the tool chain environment.
@@ -274,11 +256,11 @@ Specifically, you should verify that all arguments are parsed correctly, e.g. by
xfel-calibrate DETECTOR NOTEBOOK_TYPE --help
-From then on, check include if parallel slurm jobs are exectuted correctly and if a report
+From then on, check include if parallel slurm jobs are executed correctly and if a report
is generated at the end.
Finally, you should verify that the report contains the information you'd like to convey and
-is inteligable to people other than you.
+is intelligible to people other than you.
.. note::
@@ -298,4 +280,5 @@ documentation.
.. _matplotlib: https://matplotlib.org/
.. _numpy: http://www.numpy.org/
.. _h5py: https://www.h5py.org/
-.. _iCalibrationDB: https://in.xfel.eu/readthedocs/docs/icalibrationdb/en/latest/
+.. _iCalibrationDB: https://git.xfel.eu/detectors/cal_db_interactive
+.. _extra_data: https://extra-data.readthedocs.io/en/latest/
\ No newline at end of file
diff --git a/docs/source/xfel_calibrate_conf.rst b/docs/source/xfel_calibrate_conf.rst
new file mode 100644
index 0000000000000000000000000000000000000000..4fed8d2650a0ab6a4713ac853392890976d8bc39
--- /dev/null
+++ b/docs/source/xfel_calibrate_conf.rst
@@ -0,0 +1,6 @@
+xfel_calibrate
+==============
+
+.. module:: xfel_calibrate.calibrate
+
+.. autofunction:: balance_sequences
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index ab99edd4768548e7dd696fa55cb6efa7c1dccd65..f8e99656a9694b6776c78ea8b74a5ba8daf2f2f8 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -17,33 +17,30 @@
"metadata": {},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/SPB/202131/p900230/raw\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/agipd_resolve_conf\" # the folder to output to, required\n",
+ "in_folder = \"/gpfs/exfel/exp/MID/202201/p002834/raw\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/esobolev/pycal_litfrm/p002834/r0225\" # the folder to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
"train_ids = [-1] # train IDs to correct, set to -1 for all, range allowed\n",
- "run = 275 # runs to process, required\n",
+ "run = 225 # runs to process, required\n",
"\n",
- "karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
+ "karabo_id = \"MID_DET_AGIPD1M-1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
"receiver_template = \"{}CH0\" # inset for receiver devices\n",
"path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
"instrument_source_template = '{}/DET/{}:xtdf' # path in the HDF5 file to images\n",
"index_source_template = 'INDEX/{}/DET/{}:xtdf/' # path in the HDF5 file to images\n",
"ctrl_source_template = '{}/MDL/FPGA_COMP' # path to control information\n",
- "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
+ "karabo_id_control = \"MID_EXP_AGIPD1M1\" # karabo-id for control device\n",
"\n",
- "slopes_ff_from_files = \"\" # Path to locally stored SlopesFF and BadPixelsFF constants\n",
+ "slopes_ff_from_files = \"\" # Path to locally stored SlopesFF and BadPixelsFF constants, loaded in precorrection notebook\n",
"\n",
"use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
"cal_db_timeout = 30000 # in milliseconds\n",
"creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
"\n",
- "use_ppu_device = '' # Device ID for a pulse picker device to only process picked trains, empty string to disable\n",
- "ppu_train_offset = 0 # When using the pulse picker, offset between the PPU's sequence start and actually picked train\n",
- "\n",
"mem_cells = 0 # Number of memory cells used, set to 0 to automatically infer\n",
"bias_voltage = 0 # bias voltage, set to 0 to use stored value in slow data.\n",
"acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
@@ -85,15 +82,27 @@
"melt_snow = False # Identify (and optionally interpolate) 'snowy' pixels\n",
"mask_zero_std = False # Mask pixels with zero standard deviation across train\n",
"low_medium_gap = False # 5 sigma separation in thresholding between low and medium gain\n",
+ "round_photons = False # Round to absolute number of photons, only use with gain corrections\n",
"\n",
- "use_litframe_device = '' # Device ID for a lit frame finder device to only process illuminated frames, empty string to disable\n",
+ "# Optional auxiliary devices\n",
+ "use_ppu_device = '' # Device ID for a pulse picker device to only process picked trains, empty string to disable\n",
+ "ppu_train_offset = 0 # When using the pulse picker, offset between the PPU's sequence start and actually picked train\n",
+ "\n",
+ "use_litframe_finder = 'off' # Process only illuminated frames: 'off' - disable, 'device' - use online device data, 'offline' - use offline algorithm, 'auto' - choose online/offline source automatically (default)\n",
+ "litframe_device_id = '' # Device ID for a lit frame finder device, empty string to auto detection\n",
"energy_threshold = -1000 # The low limit for the energy (uJ) exposed by frames subject to processing. If -1000, selection by pulse energy is disabled\n",
"\n",
+ "use_xgm_device = '' # DoocsXGM device ID to obtain actual photon energy, operating condition else.\n",
+ "\n",
+ "# Output parameters\n",
+ "recast_image_data = '' # Cast data to a different dtype before saving\n",
+ "compress_fields = ['gain', 'mask'] # Datasets in image group to compress.\n",
+ "\n",
"# Plotting parameters\n",
"skip_plots = False # exit after writing corrected files and metadata\n",
"cell_id_preview = 1 # cell Id used for preview in single-shot plots\n",
"\n",
- "# Paralellization parameters\n",
+ "# Parallelization parameters\n",
"chunk_size = 1000 # Size of chunk for image-wise correction\n",
"n_cores_correct = 16 # Number of chunks to be processed in parallel\n",
"n_cores_files = 4 # Number of files to be processed in parallel\n",
@@ -215,6 +224,7 @@
" corr_bools[\"melt_snow\"] = melt_snow\n",
" corr_bools[\"mask_zero_std\"] = mask_zero_std\n",
" corr_bools[\"low_medium_gap\"] = low_medium_gap\n",
+ " corr_bools[\"round_photons\"] = round_photons\n",
"\n",
"# Many corrections don't apply to fixed gain mode; will explicitly disable later if detected\n",
"disable_for_fixed_gain = [\n",
@@ -441,16 +451,40 @@
"metadata": {},
"outputs": [],
"source": [
- "if use_litframe_device:\n",
- " # check run for the AgipdLitFrameFinder device\n",
+ "if use_litframe_finder != 'off':\n",
+ " from extra_redu import make_litframe_finder, LitFrameFinderError\n",
+ " from extra_redu.litfrm.utils import litfrm_run_report\n",
"\n",
- " if use_litframe_device + ':output' in dc.instrument_sources:\n",
- " # Use selection provided by the AgipdLitFrameFinder (if the device is recorded)\n",
- " cell_sel = LitFrameSelection(use_litframe_device, dc, train_ids, max_pulses, energy_threshold)\n",
- " train_ids = cell_sel.train_ids\n",
- " else:\n",
- " # Use range selection (if the device is not recorded)\n",
- " print(f\"WARNING: LitFrameFinder {use_litframe_device} device is not found.\")\n",
+ " if use_litframe_finder not in ['auto', 'offline', 'online']:\n",
+ " raise ValueError(\"Unexpected value in 'use_litframe_finder'.\")\n",
+ "\n",
+ " inst = karabo_id_control[:3]\n",
+ " litfrm = make_litframe_finder(inst, dc, litframe_device_id)\n",
+ " try:\n",
+ " if use_litframe_finder == 'auto':\n",
+ " r = litfrm.read_or_process()\n",
+ " elif use_litframe_finder == 'offline':\n",
+ " r = litfrm.process()\n",
+ " elif use_litframe_finder == 'online':\n",
+ " r = litfrm.read()\n",
+ "\n",
+ " report = litfrm_run_report(r)\n",
+ " print(\"Lit-frame patterns:\")\n",
+ " print(\" # trains Np Nd Nf lit frames\")\n",
+ " for rec in report:\n",
+ " frmintf = ', '.join(\n",
+ " [':'.join([str(n) for n in slc]) for slc in rec['frames']]\n",
+ " )\n",
+ " trsintf = ':'.join([str(n) for n in rec['trains']])\n",
+ " print(\n",
+ " (\"{pattern_no:2d} {trsintf:25s} {npulse:4d} \"\n",
+ " \"{ndataframe:3d} {nframe:3d} [{frmintf}]\"\n",
+ " ).format(frmintf=frmintf, trsintf=trsintf, **rec)\n",
+ " )\n",
+ " cell_sel = LitFrameSelection(r, train_ids, max_pulses, energy_threshold)\n",
+ " except LitFrameFinderError as err:\n",
+ " print(\"Cannot use AgipdLitFrameFinder due to:\")\n",
+ " print(err)\n",
" cell_sel = CellRange(max_pulses, max_cells=mem_cells)\n",
"else:\n",
" # Use range selection\n",
@@ -459,6 +493,35 @@
"print(cell_sel.msg())"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "actual_photon_energy = None\n",
+ "\n",
+ "if use_xgm_device:\n",
+ " # Try to obtain photon energy from XGM device.\n",
+ " wavelength_data = dc[use_xgm_device, 'pulseEnergy.wavelengthUsed']\n",
+ " \n",
+ " try:\n",
+ " from scipy.constants import h, c, e\n",
+ " \n",
+ " # Read wavelength as a single value and convert to hv.\n",
+ " actual_photon_energy = (h * c / e) / (wavelength_data.as_single_value(rtol=1e-2) * 1e-6)\n",
+ " print(f'Obtained actual photon energy {actual_photon_energy:.3f} keV from {use_xgm_device}')\n",
+ " except ValueError:\n",
+ " if round_photons:\n",
+ " print('WARNING: XGM source available but actual photon energy varies greater than 1%, '\n",
+ " 'photon rounding disabled!')\n",
+ " round_photons = False\n",
+ "\n",
+ "if actual_photon_energy is None and round_photons:\n",
+ " print('WARNING: Using operating condition for actual photon energy in photon rounding mode, this is NOT reliable!')\n",
+ " actual_photon_energy = photon_energy"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -492,7 +555,12 @@
"agipd_corr.cm_dark_fraction = cm_dark_fraction\n",
"agipd_corr.cm_n_itr = cm_n_itr\n",
"agipd_corr.noisy_adc_threshold = noisy_adc_threshold\n",
- "agipd_corr.ff_gain = ff_gain"
+ "agipd_corr.ff_gain = ff_gain\n",
+ "agipd_corr.actual_photon_energy = actual_photon_energy\n",
+ "\n",
+ "agipd_corr.compress_fields = compress_fields\n",
+ "if recast_image_data:\n",
+ " agipd_corr.recast_image_fields['data'] = np.dtype(recast_image_data)"
]
},
{
@@ -672,6 +740,7 @@
"\n",
" img_counts = pool.map(agipd_corr.apply_selected_pulses, range(len(file_batch)))\n",
" step_timer.done_step(\"Applying selected cells after common mode correction\")\n",
+ " \n",
" # Perform image-wise correction\"\n",
" pool.starmap(agipd_corr.gain_correction, imagewise_chunks(img_counts))\n",
" step_timer.done_step(\"Gain corrections\")\n",
@@ -817,10 +886,11 @@
" tid, data = run_data.select(f'{detector_id}/DET/*', source).train_from_id(tid)\n",
" else:\n",
" tid, data = next(iter(run_data.select(f'{detector_id}/DET/*', source).trains(require_all=True)))\n",
- "\n",
+ " \n",
" # TODO: remove and use the keep_dims version after updating Extra-data.\n",
" # Avoid using default axis with sources of an expected scalar value per train.\n",
- " if len(range(*cell_sel.crange)) == 1 and source in ['image.blShift', 'image.cellId', 'image.pulseId']:\n",
+ " nfrm = cell_sel.get_cells_on_trains([tid]).sum()\n",
+ " if nfrm == 1 and source in ['image.blShift', 'image.cellId', 'image.pulseId']:\n",
" axis = 0\n",
" else:\n",
" axis = -3\n",
@@ -828,10 +898,8 @@
" stacked_data = stack_detector_data(\n",
" train=data, data=source, fillvalue=fillvalue, modules=modules, axis=axis)\n",
" # Add cellId dimension when correcting one cellId only.\n",
- " if (\n",
- " len(range(*cell_sel.crange)) == 1 and\n",
- " data_folder != run_folder # avoid adding pulse dims for raw data.\n",
- " ):\n",
+ " # avoid adding pulse dims for raw data.\n",
+ " if (nfrm == 1 and data_folder != run_folder):\n",
" stacked_data = stacked_data[np.newaxis, ...]\n",
"\n",
" return tid, stacked_data"
@@ -946,7 +1014,9 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "scrolled": false
+ },
"outputs": [],
"source": [
"pulse_range = [np.min(pulseId[pulseId>=0]), np.max(pulseId[pulseId>=0])]\n",
@@ -1257,7 +1327,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.12"
+ "version": "3.8.11"
}
},
"nbformat": 4,
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
index fab1aa7971775019c7266c2607581ce6a368e030..7c49c3c895e4672d4f4c00e8c48fa4dddf0c8026 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
@@ -65,12 +65,13 @@
"sigma_limit = 0. # If >0, repeat fit keeping only bins within mu +- sigma_limit*sigma\n",
"\n",
"# Detector conditions\n",
- "max_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
- "bias_voltage = 300 # Bias voltage\n",
- "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
- "gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
- "photon_energy = 8.05 # photon energy in keV\n",
- "integration_time = -1 # integration time, negative values for auto-detection."
+ "# NOTE: The below parameters are needed for the summary notebook when running through xfel-calibrate.\n",
+ "mem_cells = -1 # number of memory cells used, negative values for auto-detection. \n",
+ "bias_voltage = 300 # Bias voltage. \n",
+ "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine.\n",
+ "gain_setting = -1 # the gain setting, negative values for auto-detection.\n",
+ "photon_energy = 8.05 # photon energy in keV.\n",
+ "integration_time = -1 # integration time, negative values for auto-detection."
]
},
{
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
index 5eb1eb2b89eecac949eb539eebc9574a7395175f..fd9a711612d18f42fac4fe391b788f37dcf1df4c 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
@@ -51,12 +51,12 @@
"\n",
"\n",
"# Detector conditions\n",
- "max_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
- "bias_voltage = 0. # Bias voltage\n",
- "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
- "gain_setting = -1 # the gain setting, use 0.1 to try to auto-determine\n",
- "photon_energy = 8.05 # photon energy in keV\n",
- "integration_time = -1 # integration time, negative values for auto-detection."
+ "mem_cells = -1 # number of memory cells used, negative values for auto-detection.\n",
+ "bias_voltage = 0. # Bias voltage\n",
+ "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
+ "gain_setting = -1 # the gain setting, negative values for auto-detection.\n",
+ "photon_energy = 8.05 # photon energy in keV\n",
+ "integration_time = -1 # integration time, negative values for auto-detection."
]
},
{
@@ -144,17 +144,17 @@
" ctrl_src=ctrl_src,\n",
" raise_error=False, # to be able to process very old data without mosetting value\n",
")\n",
- "\n",
- "mem_cells = agipd_cond.get_num_cells()\n",
+ "if mem_cells < 0:\n",
+ " mem_cells = agipd_cond.get_num_cells()\n",
"if mem_cells is None:\n",
" raise ValueError(f\"No raw images found in {run_folder}\")\n",
"if acq_rate == 0.:\n",
" acq_rate = agipd_cond.get_acq_rate()\n",
- "if gain_setting == -1:\n",
+ "if gain_setting < 0:\n",
" gain_setting = agipd_cond.get_gain_setting(creation_time)\n",
"if bias_voltage == 0.:\n",
" bias_voltage = agipd_cond.get_bias_voltage(karabo_id_control)\n",
- "if integration_time == -1:\n",
+ "if integration_time < 0:\n",
" integration_time = agipd_cond.get_integration_time()\n",
"\n",
"# Evaluate detector instance for mapping\n",
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index 38f726532c34b23e707e098a6fe08db72bc3613b..e739991f81f64a6e20b455700b2f4f1a5674e899 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -6,13 +6,13 @@
"source": [
"# Characterize AGIPD Pulse Capacitor Data #\n",
"\n",
- "Author: S. Hauf, Version 1.0\n",
+ "Author: Detector Operations Group, Version 1.0\n",
"\n",
"The following code characterizes AGIPD gain via data take with the pulse capacitor source (PCS). The PCS allows scanning through the high and medium gains of AGIPD, by subsequently intecreasing the number of charge pulses from a on-ASIC capicitor, thus increasing the charge a pixel sees in a given integration time.\n",
"\n",
"Because induced charge does not originate from X-rays on the sensor, the gains evaluated here will later need to be rescaled with gains deduced from X-ray data.\n",
"\n",
- "PCS data is organized into multiple runs, as the on-ASIC current source cannot supply all pixels of a given module with charge at the same time. Hence, only certain pixel rows will have seen charge for a given image. These rows then first need to be combined into single module images again.\n",
+ "PCS data is organized into multiple runs, as the on-ASIC current source cannot supply all pixels of a given module with charge at the same time. Hence, only certain pixel rows will have seen charge for a given image. These rows then first need to be combined into single module images again. This script uses new style of merging, which does not support old data format.\n",
"\n",
"We then use a K-means clustering algorithm to identify components in the resulting per-pixel data series, matching to three general regions:\n",
"\n",
@@ -28,12 +28,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-27T23:10:33.721006Z",
- "start_time": "2019-07-27T23:10:33.709017Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
@@ -46,9 +41,11 @@
"modules = [-1] # modules to work on, required, range allowed\n",
"karabo_da = [\"all\"]\n",
"karabo_da_control = \"AGIPD1MCTRL00\" # karabo DA for control infromation\n",
- "karabo_id_control = \"SPB_IRU_AGIPD1M1\"\n",
+ "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for the control device e.g. \"MID_EXP_AGIPD1M1\", or \"SPB_IRU_AGIPD1M1\"\n",
"karabo_id = \"SPB_DET_AGIPD1M-1\"\n",
- "h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP' # path to control information\n",
+ "ctrl_source_template = '{}/MDL/FPGA_COMP' # path to control information\n",
+ "instrument_source_template = '{}/DET/{}:xtdf' # path in the HDF5 file to images\n",
+ "receiver_template = \"{}CH0\" # inset for receiver devices\n",
"\n",
"use_dir_creation_date = True\n",
"delta_time = 0 # offset to the creation time (e.g. useful in case we want to force the system to use diff. dark constants)\n",
@@ -56,27 +53,25 @@
"local_output = True # output constants locally\n",
"db_output = False # output constants to database\n",
"\n",
- "bias_voltage = 300 # detector bias voltage\n",
- "mem_cells = 0. # number of memory cells used, use 0 to auto-derive\n",
- "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
- "gain_setting = 0.1 # gain setting can have value 0 or 1, Default=0.1 for no (None) gain-setting\n",
- "integration_time = -1 # integration time, negative values for auto-detection.\n",
+ "bias_voltage = -1 # detector bias voltage, negative values for auto-detection.\n",
+ "mem_cells = -1 # number of memory cells used, negative values for auto-detection.\n",
+ "acq_rate = -1. # the detector acquisition rate, negative values for auto-detection.\n",
+ "gain_setting = -1 # gain setting can have value 0 or 1, negative values for auto-detection.\n",
+ "integration_time = -1 # integration time, negative values for auto-detection.\n",
+ "FF_gain = 7.3 # ADU/keV, gain from FF to convert ADU to keV\n",
+ "\n",
+ "fit_hook = False # fit a hook function to medium gain slope --> run without hook\n",
+ "high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. Runtime: ~ 1h\n",
"\n",
- "interlaced = False # assume interlaced data format, for data prior to Dec. 2017\n",
- "fit_hook = True # fit a hook function to medium gain slope\n",
- "rawversion = 2 # RAW file format version\n",
- "high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. Runtime: ~ 1h"
+ "hg_range = [30,210]#[0,-150] # range for linear fit. If upper edge is negative use clustering result (bound_hg - 20)\n",
+ "mg_range = [-277,600]#[-350,600] # range for linear fit. If lower edge is negative use clustering result (bound_mg + 20)\n",
+ "sigma_dev_cut = 5.0 # parameters outside the range median +- sigma_dev_cut*MAD are replaced with the median"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-27T23:10:37.540374Z",
- "start_time": "2019-07-27T23:10:34.428000Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"# imports, usually no need to change anything here\n",
@@ -95,13 +90,17 @@
"from ipyparallel import Client\n",
"\n",
"import matplotlib.pyplot as plt\n",
+ "from matplotlib import gridspec\n",
+ "from matplotlib.colors import LogNorm, PowerNorm\n",
+ "import matplotlib.patches as patches\n",
+ "from mpl_toolkits.axes_grid1 import AxesGrid\n",
"\n",
"%matplotlib inline\n",
"\n",
"import XFELDetAna.xfelpyanatools as xana\n",
- "from cal_tools.agipdlib import (\n",
- " get_acq_rate, get_gain_setting, get_integration_time, get_num_cells\n",
- ")\n",
+ "from extra_data import RunDirectory\n",
+ "\n",
+ "from cal_tools.agipdlib import AgipdCtrl\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import plot_badpix_3d, show_overview\n",
"from cal_tools.tools import (\n",
@@ -113,29 +112,31 @@
" get_report,\n",
" module_index_to_qm,\n",
" parse_runs,\n",
- " run_prop_seq_from_path,\n",
" send_to_db,\n",
")\n",
"from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
"view = Client(profile=cluster_profile)[:]\n",
- "view.use_dill()\n",
- "\n",
- "IL_MODE = interlaced \n",
- "maxcells = mem_cells if not interlaced else mem_cells*2\n",
+ "view.use_dill()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"cells = mem_cells\n",
"path_temp = in_folder+\"/r{:04d}/\"\n",
"image_name_temp = 'RAW-R{:04d}-AGIPD{:02d}-S{:05d}.h5'\n",
- "seqs = n_sequences\n",
"print(\"Parameters are:\")\n",
- "print(\"Memory cells: {}/{}\".format(cells, maxcells))\n",
+ "if mem_cells < 0:\n",
+ " print(\"Memory cells: auto-detection on\")\n",
+ "else:\n",
+ " print(\"Memory cells set by user: {}\".format(mem_cells))\n",
"print(\"Runs: {}\".format(runs))\n",
"print(\"Modules: {}\".format(modules))\n",
- "print(\"Sequences: {}\".format(seqs))\n",
- "print(\"Interlaced mode: {}\".format(IL_MODE))\n",
- "\n",
- "run, prop, seq = run_prop_seq_from_path(in_folder)\n",
"\n",
"instrument = karabo_id.split(\"_\")[0]\n",
"\n",
@@ -157,260 +158,214 @@
]
},
{
- "cell_type": "markdown",
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
- "## Read in data and merge ##\n",
- "\n",
- "The number of bursts in each sequence file is determined from the sequence files of the first module."
+ "first_run = runs[0]\n",
+ "channel = 0\n",
+ "ctrl_src = ctrl_source_template.format(karabo_id_control)\n",
+ "instrument_src = instrument_source_template.format(karabo_id, receiver_template).format(channel)\n",
+ "\n",
+ "agipd_cond = AgipdCtrl(\n",
+ " run_dc=RunDirectory(f'{in_folder}/r{first_run:04d}'),\n",
+ " image_src=instrument_src,\n",
+ " ctrl_src=ctrl_src,\n",
+ " raise_error=False, # to be able to process very old data without gain_setting value\n",
+ ")"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-27T23:10:57.019242Z",
- "start_time": "2019-07-27T23:10:37.542473Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
- "run = runs[0]\n",
- "bursts_per_file = []\n",
- "channel = 0\n",
- "control_fname = f'{in_folder}/r{run:04d}/RAW-R{run:04d}-{karabo_da_control}-S00000.h5'\n",
- "slow_paths = (control_fname, karabo_id_control)\n",
- "\n",
- "for seq in range(seqs):\n",
- " fname = os.path.join(path_temp.format(run),\n",
- " image_name_temp.format(run, channel, seq))\n",
- " print('Reading ',fname)\n",
- " \n",
- " if acq_rate == 0.:\n",
- " acq_rate = get_acq_rate(\n",
- " fast_paths=(fname, karabo_id, channel), slow_paths=slow_paths)\n",
- " print(\"Acquisition rate set from file: {} MHz\".format(acq_rate))\n",
- "\n",
- " if mem_cells == 0:\n",
- " cells = get_num_cells(fname, karabo_id, channel)\n",
- " maxcells = cells\n",
- " mem_cells = cells # avoid setting twice\n",
- " print(\"Memory cells set from file: {}\".format(cells))\n",
- " \n",
- " f = h5py.File(fname, 'r')\n",
- " if rawversion == 2:\n",
- " count = np.squeeze(f[\"/INDEX/{}/DET/{}CH0:xtdf/image/count\".format(karabo_id, channel)]) \n",
- " bursts_per_file.append(np.count_nonzero(count))\n",
- " else:\n",
- " status = np.squeeze(f[\"/INDEX/{}/DET/{}CH0:xtdf/image/status\".format(karabo_id, channel)]) \n",
- " bursts_per_file.append(np.count_nonzero(status != 0))\n",
- " f.close()\n",
- "bursts_per_file = np.array(bursts_per_file)\n",
- "print(\"Bursts per sequence file are: {}\".format(bursts_per_file))\n",
- "\n",
"# Define creation time\n",
- "creation_time=None\n",
+ "creation_time = None\n",
"if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run)\n",
+ " creation_time = get_dir_creation_date(in_folder, first_run)\n",
" creation_time = creation_time + timedelta(hours=delta_time)\n",
- "print(f\"Using {creation_time} as creation time of constant.\")\n",
"\n",
- "if not creation_time and use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run)\n",
+ "# Read AGIPD parameter conditions.\n",
+ "if acq_rate < 0.:\n",
+ " acq_rate = agipd_cond.get_acq_rate()\n",
+ "if mem_cells < 0:\n",
+ " mem_cells = agipd_cond.get_num_cells()\n",
+ "# TODO: look for alternative for passing creation_time\n",
+ "if gain_setting < 0:\n",
+ " gain_setting = agipd_cond.get_gain_setting(creation_time)\n",
+ "if bias_voltage < 0:\n",
+ " bias_voltage = agipd_cond.get_bias_voltage(karabo_id_control)\n",
+ "if integration_time < 0:\n",
+ " integration_time = agipd_cond.get_integration_time()\n",
"\n",
- "if creation_time:\n",
- " print(\"Using {} as creation time\".format(creation_time.isoformat()))"
+ "print(f\"Acquisition rate: {acq_rate} MHz\")\n",
+ "print(f\"Memory cells: {mem_cells}\")\n",
+ "print(f\"Gain setting: {gain_setting}\")\n",
+ "print(f\"Integration time: {integration_time}\")\n",
+ "print(f\"Using {creation_time} as creation time of constant.\")"
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
"source": [
- "if \"{\" in h5path_ctrl:\n",
- " h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
- "\n",
- "if gain_setting == 0.1:\n",
- " if creation_time.replace(tzinfo=None) < dateutil.parser.parse('2020-01-31'):\n",
- " print(\"Set gain-setting to None for runs taken before 2020-01-31\")\n",
- " gain_setting = None\n",
- " else:\n",
- " try:\n",
- " gain_setting = get_gain_setting(control_fname, h5path_ctrl)\n",
- " except Exception as e:\n",
- " print(f'Error while reading gain setting from: \\n{control_fname}')\n",
- " print(e)\n",
- " print(\"Gain setting is not found in the control information\")\n",
- " print(\"Data will not be processed\")\n",
- " sequences = []\n",
- "print(f\"Gain setting: {gain_setting}\")\n",
+ "## Read in data and merge ##\n",
"\n",
- "if integration_time < 0:\n",
- " integration_time = get_integration_time(control_fname, h5path_ctrl)\n",
- "print(f\"Integration time: {integration_time}\")"
+ "The new merging cannot handle the old data formats before Dec. 2017."
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-27T23:33:49.080256Z",
- "start_time": "2019-07-27T23:10:57.021963Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
- "def read_and_merge_module_data(cells, path_temp, image_name_temp,\n",
- " runs, seqs, il_mode, rawversion, instrument, channel):\n",
- " import os\n",
- "\n",
- " import h5py\n",
- " import numpy as np\n",
- "\n",
+ "def merge_data(runs, in_folder, karabo_id, channel):\n",
" \n",
- " def cal_bursts_per_file(run, dseq=0):\n",
- "\n",
- " bursts_per_file = []\n",
- " channel = 0\n",
- "\n",
- " for seq in range(dseq, seqs+dseq):\n",
- " #print(run, channel, seq)\n",
- " fname = os.path.join(path_temp.format(run),\n",
- " image_name_temp.format(run, channel, seq))\n",
- " #print('Reading ',fname)\n",
- " with h5py.File(fname, 'r') as f:\n",
- " if rawversion == 2:\n",
- " count = np.squeeze(f[\"/INDEX/{}/DET/{}CH0:xtdf/image/count\".format(karabo_id, channel)][()])\n",
- " bursts_per_file.append(np.count_nonzero(count))\n",
- " del count\n",
- " else:\n",
- " status = np.squeeze(f[\"/INDEX/{}/DET/{}CH0:xtdf/image/status\".format(karabo_id, channel)][()]) \n",
- " bursts_per_file.append(np.count_nonzero(status != 0))\n",
- " del status\n",
- " if bursts_per_file[0] == 0:\n",
- " return cal_bursts_per_file(run, dseq=dseq+1) # late start of daq\n",
- " return np.array(bursts_per_file), dseq\n",
- " \n",
- " #bursts_per_file = np.hstack([0, bursts_per_file])\n",
+ " in_folder = in_folder+\"/r{:04d}/\"\n",
+ " def count_min_bursts(in_folder, runs, channel):\n",
" \n",
- " bursts_total = np.max([np.sum(cal_bursts_per_file(run)[0]) for run in runs])\n",
+ " bursts = np.zeros(len(runs))\n",
+ " for i, run in enumerate(runs):\n",
+ " run_folder_path = in_folder.format(run)\n",
+ " r = RunDirectory(run_folder_path)\n",
+ " instrument_source = karabo_id+'/DET/{}CH0:xtdf'.format(channel)\n",
+ " c = r.get_array(instrument_source, 'image.length')\n",
+ " total_frame = np.count_nonzero(c)\n",
+ " cells = r.detector_info(instrument_source)['frames_per_train']\n",
+ " bursts[i] = total_frame // cells\n",
+ " bursts_total = np.min(bursts).astype(int)\n",
+ "\n",
+ " return bursts_total, cells\n",
" \n",
- " cfac = 2 if il_mode else 1\n",
- " \n",
- " def read_raw_data_file(fname, channel, cells = cells, cells_tot = cells, bursts = 250,\n",
- " skip_first_burst = True, first_burst_length = cells):\n",
- " data = None\n",
- " cellID_all = None\n",
- " with h5py.File(fname, 'r') as f:\n",
- " \n",
- " #print('Reading ',fname)\n",
- " image_path_temp = 'INSTRUMENT/{}/DET/{}CH0:xtdf/image/data'.format(karabo_id, channel)\n",
- " cellID_path_temp = 'INSTRUMENT/{}/DET/{}CH0:xtdf/image/cellId'.format(karabo_id, channel)\n",
- " if rawversion == 2:\n",
- " count = np.squeeze(f[\"/INDEX/{}/DET/{}CH0:xtdf/image/count\".format(karabo_id, channel)])\n",
- " first = np.squeeze(f[\"/INDEX/{}/DET/{}CH0:xtdf/image/first\".format(karabo_id, channel)])\n",
- " last_index = int(first[count != 0][-1]+count[count != 0][-1])\n",
- " first_index = int(first[count != 0][0])\n",
- " else:\n",
- " status = np.squeeze(f[\"/INDEX/{}/DET/{}CH0:xtdf/image/status\".format(karabo_id, channel)])\n",
- " if np.count_nonzero(status != 0) == 0:\n",
- " return\n",
- " last = np.squeeze(f[\"/INDEX/{}/DET/{}CH0:xtdf/image/last\".format(karabo_id, channel)])\n",
- " last_index = int(last[status != 0][-1])\n",
- " first_index = int(last[status != 0][0])\n",
- " #print(first_index, last_index)\n",
- " data = f[image_path_temp][first_index:last_index,...][()]\n",
- "\n",
- " cellID_all = np.squeeze(f[cellID_path_temp][first_index:last_index,...][()])\n",
- " data = data[cellID_all<cells, ...]\n",
- " \n",
- " #bursts = int(data.shape[0]/adcells)\n",
- " #print('Bursts: ', bursts)\n",
- " analog = np.zeros((bursts - skip_first_burst, cells//cfac, 128, 512))\n",
- " digital = np.zeros((bursts - skip_first_burst, cells//cfac, 128, 512))\n",
- " cellID = np.zeros(( (bursts - skip_first_burst) * cells))\n",
- " offset = skip_first_burst * first_burst_length\n",
- " \n",
- " for b in range(min(bursts, data.shape[0]//cells-1) - skip_first_burst-1):\n",
- " try:\n",
- " \n",
- " analog[b, : cells//cfac, ...] = np.swapaxes(data[b * cells_tot + offset : b * cells_tot + cells + offset : cfac,\n",
- " 0, ...], -1, -2)\n",
- " digital[b, : cells//cfac, ...] = np.swapaxes(data[b * cells_tot + cfac - 1 + skip_first_burst * first_burst_length : \n",
- " b * cells_tot + cells + cfac - 1 + offset :cfac, cfac%2, ...], -1, -2)\n",
- "\n",
- " cellID[ b * cells : (b + 1) * cells] = cellID_all[b * cells_tot + offset : b * cells_tot + cells + offset].flatten()\n",
- " except:\n",
- " #print(b * cells_tot + offset, b * cells_tot + cells + offset)\n",
- " #print(b, offset, cells, data.shape[0]//cells)\n",
- " raise AttributeError(\"Foo\")\n",
- " return {'analog': analog, 'digital': digital, 'cellID': cellID}\n",
+ " bursts_total, cells = count_min_bursts(in_folder, runs, channel)\n",
"\n",
- " \n",
- " pc_data = {'analog': np.zeros((bursts_total, cells//cfac, 128, 512)),\n",
- " 'digital': np.zeros((bursts_total, cells//cfac, 128, 512)),\n",
- " 'cellID': np.zeros(((bursts_total) * cells))\n",
- " }\n",
- " pc_data_merged = {'analog': np.zeros((bursts_total, cells//cfac, 128, 512)),\n",
- " 'digital': np.zeros((bursts_total, cells//cfac, 128, 512)),\n",
+ " pc_data = {'analog': np.zeros((bursts_total, cells, 128, 512)),\n",
+ " 'digital': np.zeros((bursts_total, cells, 128, 512)),\n",
" 'cellID': np.zeros(((bursts_total) * cells))\n",
" }\n",
- " \n",
- " for run_idx, run in enumerate(runs):\n",
- " bursts_per_file, dseq = cal_bursts_per_file(run)\n",
- " print(\"Run {}: bursts per file: {} -> {} total\".format(run, bursts_per_file, np.sum(bursts_per_file)))\n",
- " #Read files in\n",
- " last_burst = 0\n",
- " for seq in range(dseq, seqs+dseq):\n",
- " fname = os.path.join(path_temp.format(run),\n",
- " image_name_temp.format(run, channel, seq))\n",
- " if seq-dseq == 0:\n",
- " skip_first_burst = True\n",
- " else:\n",
- " skip_first_burst = False\n",
- " bursts = bursts_per_file[seq-dseq]\n",
- " \n",
- " try:\n",
- " aa = read_raw_data_file(fname, channel, bursts = bursts,\n",
- " skip_first_burst = skip_first_burst,\n",
- " first_burst_length = cells)\n",
- " pc_data['analog'][last_burst : last_burst+bursts_per_file[seq-dseq]-skip_first_burst, ...] = aa['analog']\n",
- " pc_data['digital'][last_burst : last_burst+bursts_per_file[seq-dseq]-skip_first_burst, ...] = aa['digital']\n",
- " pc_data['cellID'][last_burst * cells : (last_burst+bursts_per_file[seq-dseq]-skip_first_burst) * cells, ...] = aa['cellID']\n",
- " \n",
- " except Exception as e:\n",
- " print(e)\n",
- " pc_data['analog'][last_burst : last_burst+bursts_per_file[seq-dseq]-skip_first_burst, ...] = 0\n",
- " pc_data['digital'][last_burst : last_burst+bursts_per_file[seq-dseq]-skip_first_burst, ...] = 0\n",
- " pc_data['cellID'][last_burst * cells : (last_burst+bursts_per_file[seq-dseq]-skip_first_burst) * cells, ...] = 0\n",
- " finally:\n",
- " last_burst += bursts_per_file[seq-dseq]-skip_first_burst\n",
- " # Copy injected rows\n",
- " for row_i in range(8):\n",
- " try:\n",
- " pc_data_merged['analog'][:,:,row_i * 8 + (7 - run_idx),:] = pc_data['analog'][:bursts_total,:cells//cfac,row_i * 8 + (7 - run_idx),:]\n",
- " pc_data_merged['analog'][:,:,64 + row_i * 8 + run_idx ,:] = pc_data['analog'][:bursts_total,:cells//cfac, 64 + row_i * 8 + run_idx,:]\n",
- " pc_data_merged['digital'][:,:,row_i * 8 + (7 - run_idx),:] = pc_data['digital'][:bursts_total,:cells//cfac,row_i * 8 + (7 - run_idx),:]\n",
- " pc_data_merged['digital'][:,:,64 + row_i * 8 + run_idx ,:] = pc_data['digital'][:bursts_total,:cells//cfac, 64 + row_i * 8 + run_idx,:] \n",
- " except Exception as e:\n",
- " print(e)\n",
- " #Check cellIDs\n",
- " #Copy cellIDs of first run\n",
- " if run_idx == 0:\n",
- " pc_data_merged['cellID'][...] = pc_data['cellID'][...]\n",
- " #Check cellIDs of all the other runs\n",
- " #else:\n",
- " # print('cellID difference:{}'.format(np.sum(pc_data_merged['cellID']-pc_data['cellID'])))\n",
- " return pc_data_merged['analog'], pc_data_merged['digital'], pc_data_merged['cellID']\n",
"\n",
+ " for counter, run in enumerate(runs):\n",
+ " \n",
+ " run_folder_path = in_folder.format(run)\n",
+ " print('Run: ', run_folder_path)\n",
+ " r = RunDirectory(run_folder_path)\n",
+ " instrument_source = karabo_id+'/DET/{}CH0:xtdf'.format(channel)\n",
+ " print('Module: ', instrument_source)\n",
+ " \n",
+ " d = r.get_array(instrument_source, 'image.data')\n",
+ " d = d.values.reshape((d.shape[0] // cells, cells, 2, 512, 128))\n",
+ " d = np.moveaxis(d, 4, 3)\n",
+ " c = r.get_array(instrument_source, 'image.cellId')\n",
+ " print('Bursts: {}, Mem cells: {}\\n'.format(c.shape[0] // cells, cells))\n",
+ "\n",
+ " for i in range(7 - counter, 64, 8): # first run starts in row 7 and then decreases in each run by 1 \n",
+ " pc_data['analog'][..., i, :] = d[:bursts_total, :, 0, i, :]\n",
+ " pc_data['digital'][..., i, :] = d[:bursts_total, :, 1, i, :]\n",
+ " for i in range(64 + counter, 128, 8): # separated module to 2 halves to catch the injection pattern correctly\n",
+ " pc_data['analog'][..., i, :] = d[:bursts_total, :, 0, i, :]\n",
+ " pc_data['digital'][..., i, :] = d[:bursts_total, :, 1, i, :]\n",
+ " if counter == 0: \n",
+ " pc_data['cellID'][...] = c[:bursts_total * cells, 0]\n",
+ "\n",
+ " del d\n",
+ " del c\n",
+ " \n",
+ " return pc_data['analog'], pc_data['digital'], pc_data['cellID']\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"start = datetime.now()\n",
- "p = partial(read_and_merge_module_data, maxcells, path_temp, image_name_temp,\n",
- " runs, seqs, IL_MODE, rawversion, instrument)\n",
+ "p = partial(merge_data, runs, in_folder, karabo_id)\n",
"# chunk this a bit, so that we don't overuse available memory\n",
- "res = list(map(p, modules))"
+ "res = list(map(p, modules))\n",
+ "end = datetime.now() - start\n",
+ "print('Duration of merging: ', end)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Plotting of merged PC signal "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The merged image of the injected pulse capacitor signal is plotted here for selected trains and a memory cell. \n",
+ "The left side shows images of analog signal, while the right side shows digital signal image. Rows visualize the signal in a given train."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def PC_image_plotting(res, burst_of_interest, cell_of_interest):\n",
+ " \n",
+ " fig, axs = plt.subplots(len(burst_of_interest), 2, figsize=(10, 9))\n",
+ " fig.subplots_adjust(left=0.02, bottom=0.06, right=0.95, top=0.94, wspace=0.25)\n",
+ "\n",
+ " yticks_major=np.arange(0,129,64)\n",
+ " xticks_major=np.arange(0,512,64)\n",
+ " \n",
+ " for img_type in range(0,2):\n",
+ " if img_type == 0:\n",
+ " title = 'PC signal (ADU)'\n",
+ " img_title = 'Analog'\n",
+ " scale_min = 5000\n",
+ " scale_max = 11000\n",
+ " else:\n",
+ " title = 'PC gain signal (ADU)'\n",
+ " img_title = 'Digital'\n",
+ " scale_min = 4500\n",
+ " scale_max = 8500\n",
+ " \n",
+ " for x, b, in enumerate(burst_of_interest):\n",
+ " im1 = axs[x, img_type].imshow(res[0][img_type][b, cell_of_interest, ...], interpolation='nearest', \n",
+ " cmap='jet', origin='lower',vmin=scale_min, vmax=scale_max, aspect='equal')\n",
+ " cbar = fig.colorbar(im1, ax=axs[x, img_type], fraction=0.1, pad=0.2, orientation='horizontal')\n",
+ " cbar.set_label(title)\n",
+ " axs[x, img_type].set_title(\"{}, Burst: {}, Cell: {}\".format(img_title, b, cell_of_interest))\n",
+ " axs[x,img_type].set_xticks(xticks_major)\n",
+ " axs[x,img_type].xaxis.grid(True, which='major', linewidth=0.5, color='grey')\n",
+ " axs[x,img_type].set_yticks(yticks_major)\n",
+ " axs[x,img_type].yaxis.grid(True, which='major', linewidth=0.5, color='grey')\n",
+ " axs[x,img_type].set_xlabel('Columns')\n",
+ " axs[x,img_type].set_ylabel('Rows')\n",
+ " plt.show()\n",
+ " return fig"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### PC signal of a selected cell and bursts"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "burst_of_interest = [100, 200, 550]\n",
+ "cell_of_interest = 4\n",
+ "for module in modules:\n",
+ " fig = PC_image_plotting(res, burst_of_interest, cell_of_interest)"
]
},
{
@@ -427,52 +382,25 @@
"\n",
" $$y = mx + b$$\n",
" \n",
- "* for the second and third region a composite function of the form:\n",
+ "* for the second and third region a composite function (so-called hook function) of the form:\n",
" \n",
" $$y = A*e^{-(x-O)/C}+mx+b$$\n",
" \n",
- " is fitted, covering both the transition region and the medium gain slope."
+ " is fitted, covering both the transition region and the medium gain slope. \n",
+ "\n",
+ "If variable {fit_hook} is set to False (default) only the medium gain region is fitted with linear function and the transition region is omitted in the fitting procedure.\n",
+ "\n",
+ "Clustering results are not used if {hg_range} or {mg_range} are defined as positive integers, thus setting borders for the high gain region (hg_range) or medium gain region (mg_region). Data inside each region are fitted with linear function. Hook function fit to the data is not allowed in this case."
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-27T23:33:49.168464Z",
- "start_time": "2019-07-27T23:33:49.082261Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"from iminuit import Minuit\n",
"from iminuit.util import describe, make_func_code\n",
- "from sklearn.cluster import KMeans\n",
- "\n",
- "\n",
- "def calc_m_cluster(x, y):\n",
- " scan_range = 15\n",
- " ms = np.zeros((x.shape[0], scan_range))\n",
- " for i in range(scan_range):\n",
- " xdiffs = x - np.roll(x, i+1)\n",
- " ydiffs = y - np.roll(y, i+1)\n",
- " m = ydiffs/xdiffs\n",
- " ms[:,i] = m\n",
- " m = np.mean(ms, axis=1)\n",
- " \n",
- " k = KMeans(n_clusters=3, n_jobs=-2)\n",
- " k.fit(m.reshape(-1, 1)) \n",
- " ms = []\n",
- " for lbl in np.unique(k.labels_):\n",
- " xl = x[k.labels_ == lbl]\n",
- " xd = np.reshape(xl, (len(xl), 1))\n",
- " xdiff = xd - xd.transpose()\n",
- " \n",
- " yl = y[k.labels_ == lbl]\n",
- " yd = np.reshape(yl, (len(yl), 1))\n",
- " ydiff = yd - yd.transpose()\n",
- " ms.append(np.mean(np.nanmean(ydiff/xdiff, axis=0))) \n",
- " return ms, k.labels_, k.cluster_centers_\n",
"\n",
"def rolling_window(a, window):\n",
" shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)\n",
@@ -489,7 +417,6 @@
" m = ydiffs/xdiffs\n",
" ms[:,i] = m\n",
" m = np.mean(ms, axis=1)\n",
- "\n",
" m[scan_range//2:-scan_range//2+1] = np.mean(rolling_window(m, scan_range),-1)\n",
" reg1 = m > r1\n",
" reg2 = m < r2\n",
@@ -573,12 +500,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-27T23:33:59.255056Z",
- "start_time": "2019-07-27T23:33:49.170345Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"from cal_tools.tools import get_constant_from_db_and_time\n",
@@ -629,13 +551,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-27T23:34:22.590279Z",
- "start_time": "2019-07-27T23:33:59.257776Z"
- },
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"test_pixels = []\n",
@@ -653,7 +569,7 @@
" \n",
"from mpl_toolkits.axes_grid1 import ImageGrid\n",
"for mod, r in zip(modules, res):\n",
- " dig, ana, cellId = r\n",
+ " ana, dig, cellId = r \n",
" d = []\n",
" d2 = []\n",
" d3 = []\n",
@@ -666,12 +582,17 @@
" for cell in test_cells:\n",
" color = np.random.rand(3,1)\n",
" \n",
- " x = np.arange(dig.shape[0])\n",
- " y = dig[:,cell, pix[0], pix[1]]\n",
+ " \n",
+ " x = np.arange(ana.shape[0])\n",
+ " y = ana[:,cell, pix[0], pix[1]]\n",
+ " yd = dig[:,cell, pix[0], pix[1]]\n",
+ " thresh_trans = thresholds[mod][pix[0], pix[1], cell,0]\n",
+ " \n",
" \n",
" vidx = (y > 1000) & np.isfinite(y)\n",
" x = x[vidx]\n",
" y = y[vidx]\n",
+ " yd = yd[vidx]\n",
" if x.shape[0] == 0:\n",
" continue\n",
" \n",
@@ -681,24 +602,28 @@
" markers = ['o','.','x','v']\n",
" colors = ['b', 'r', 'g', 'k']\n",
" ymin = y.min()\n",
- "\n",
+ " \n",
+ " #### Transition region gain is set based on dark thresholds. ####\n",
+ " msk = yd[labels[1]] < thresh_trans\n",
+ " label_filter = np.where(labels[1])[0]\n",
+ " labels[1][label_filter[msk]] = False\n",
+ " labels[0][label_filter[msk]] = True\n",
+ " ################################################################\n",
+ " \n",
" for i, lbl in enumerate(labels):\n",
" if np.any(lbl):\n",
- " #ym = y[lbl]-y[lbl].min()\n",
" if i == 0:\n",
" gain = 0\n",
" else:\n",
" gain = 1\n",
+ " \n",
" ym = y[lbl] - offset[pix[0], pix[1], cell, gain]\n",
- " #if i != 0:\n",
- " # ym += y[labels[0]].max()-y[labels[0]].min()\n",
" h, ex, ey = np.histogram2d(x[lbl], ym, range=((0, 600), (-500, 6000)), bins=(300, 650))\n",
" H[i] += h\n",
- "\n",
- " \n",
- " \n",
- " fig = plt.figure(figsize=(10,10))\n",
+ " \n",
+ " fig = plt.figure(figsize=(10,7))\n",
" ax = fig.add_subplot(111)\n",
+ " ax.grid(lw=1.5)\n",
" for i in range(3):\n",
" H[i][H[i]==0] = np.nan \n",
" ax.imshow(H[0].T, origin=\"lower\", extent=[ex[0], ex[-1], ey[0], ey[-1]],\n",
@@ -707,8 +632,11 @@
" aspect='auto', cmap='spring', alpha=0.7, vmin=0, vmax=100)\n",
" ax.imshow(H[2].T, origin=\"lower\", extent=[ex[0], ex[-1], ey[0], ey[-1]],\n",
" aspect='auto', cmap='winter', alpha=0.7, vmin=0, vmax=1000)\n",
- " ax.set_ylabel(\"AGIPD response (ADU)\")\n",
- " ax.set_xlabel(\"PC scan point (#)\")"
+ " ax.set_ylabel(\"AGIPD response (ADU)\", fontsize=12)\n",
+ " ax.set_xlabel(\"PC scan point (#)\", fontsize=12)\n",
+ " plt.xticks(fontsize=12)\n",
+ " plt.yticks(fontsize=12)\n",
+ " plt.show()"
]
},
{
@@ -720,7 +648,7 @@
"The follwing is an visualization of the clustering and fitting for a subset of pixels. If data significantly mismatches expectations, the clustering and fitting algorithms should fail for this subset:\n",
"\n",
"* the first plot shows the clustering results for pixels which were sucessfully evaluated\n",
- "* the second plot shows the clustering results for pixels which failed to evaluate\n",
+ "* the second plot shows the clustering results for pixels which failed to evaluate--> change this \n",
"* the third plot shows the fits and fit residuals for the pixel clusters shown in the first plot\n",
"\n",
"Non-smooth behaviour is an indication that you are errorously processing interleaved data that is not, or vice versa, or have the wrong number of memory cells set.\n",
@@ -731,350 +659,294 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-27T23:34:23.910152Z",
- "start_time": "2019-07-27T23:34:22.592165Z"
- },
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
- "test_pixels = []\n",
- "tpix_range1 = [(250,254), (60,64)]\n",
- "for i in range(*tpix_range1[0]):\n",
- " for j in range(*tpix_range1[1]):\n",
- " test_pixels.append((j,i))\n",
- "test_cells = [4, 38]\n",
- "tcell = np.array(test_cells)\n",
- "tcell = tcell[tcell < mem_cells]\n",
- "if tcell.size == 0:\n",
- " test_cells = [mem_cells-1]\n",
- "else:\n",
- " test_cells = tcell.tolist()\n",
+ "def find_params(x, y, yd, thresh_trans):\n",
+ "\n",
+ " ms, labels, centers = calc_m_cluster2(x, y)\n",
+ " bound = None\n",
+ " bound_m = None\n",
" \n",
- "for mod, r in zip(modules, res):\n",
- " dig, ana, cellId = r\n",
- " d = []\n",
- " d2 = []\n",
- " d3 = []\n",
- " offset = offsets[mod]\n",
- " noise = noises[mod]\n",
- " for pix in test_pixels:\n",
- " for cell in test_cells:\n",
- " color = np.random.rand(3,1)\n",
- " \n",
- " x = np.arange(dig.shape[0])\n",
- " y = dig[:,cell, pix[0], pix[1]]\n",
+ " if labels[1].any():\n",
+ " bound = np.min(x[labels[1]])\n",
+ " bound_m = ms[1]\n",
+ " threshold = (np.mean(yd[labels[0]])+np.mean(yd[labels[2]])) / 2\n",
+ " \n",
+ " #### Transition region gain is set based on dark thresholds. ####\n",
+ " msk = yd[labels[1]] < thresh_trans\n",
+ " label_filter = np.where(labels[1])[0]\n",
+ " labels[1][label_filter[msk]] = False\n",
+ " labels[0][label_filter[msk]] = True\n",
+ " try:\n",
+ " if np.max(x[labels[0]] > 220): # step value above which dark threholds are not considered \n",
+ " labels[0][label_filter[msk]] = False\n",
+ " except ValueError:\n",
+ " print('Could not determine maximum value.')\n",
+ " pass\n",
+ " ############################################################### \n",
+ " \n",
+ " if bound is None or bound < 20: #and False:\n",
+ " ya = dig[:,cell, pix[0], pix[1]][vidx]\n",
+ " msa, labels, centers = calc_m_cluster2(x, ya, 25, -10, 25)\n",
+ " if np.count_nonzero(labels[0]) > 0:\n",
+ " bound = np.min(x[labels[0]])\n",
+ " bound_m = ms[3]\n",
+ " else:\n",
+ " bound = 0\n",
+ " bound_m = 0\n",
" \n",
- " vidx = (y > 1000) & np.isfinite(y)\n",
- " x = x[vidx]\n",
- " y = y[vidx]\n",
+ " if hg_range[1] > 0 :\n",
+ " hg_bound_high = hg_range[1]\n",
+ " hg_bound_low = hg_range[0]\n",
+ " else :\n",
+ " hg_bound_high = bound - 20\n",
+ " hg_bound_low = 0\n",
+ "\n",
+ " try:\n",
+ " if fit_hook:\n",
+ " mg_bound_high = len(x)\n",
+ " try:\n",
+ " mg_bound_low = np.max(x[labels[1]]) + 20\n",
+ " except ValueError: #raised if `y` is empty.\n",
+ " mg_bound_low = bound + 100 \n",
+ " else :\n",
+ " if mg_range[0] > 0 :\n",
+ " mg_bound_low = mg_range[0]\n",
+ " mg_bound_high = mg_range[1]\n",
" \n",
- " ms, labels, centers = calc_m_cluster2(x, y)\n",
- " bound = None\n",
- " bound_m = None\n",
- " markers = ['o','.','x','v']\n",
- " colors = ['b', 'r', 'g', 'k']\n",
- " for i, lbl in enumerate(labels):\n",
- " if i == 0:\n",
- " gain = 0\n",
- " else:\n",
- " gain = 1\n",
- " d.append({'x': x[lbl],\n",
- " 'y': y[lbl] - offset[pix[0], pix[1], cell, gain],\n",
- " 'marker': markers[i],\n",
- " 'color': colors[i],\n",
- " 'linewidth': 0\n",
- " })\n",
- " #if ms[i] < 0: # slope separating two regions\n",
- " # bound = np.min(x[lbl])\n",
- " # bound_m = ms[i]\n",
- " if labels[1].any():\n",
- " bound = np.min(x[labels[1]])\n",
- " bound_m = ms[1]\n",
- " if bound is None or bound < 20 and False:\n",
- " ya = ana[:,cell, pix[0], pix[1]][vidx]\n",
- " msa, labels, centers = calc_m_cluster2(x, ya, 25, -10, 25)\n",
- " if np.count_nonzero(labels[0]) > 0:\n",
- " \n",
- " bound = np.min(x[labels[0]])\n",
- " bound_m = ms[3]\n",
- " else:\n",
- " avg_g = np.nanmean(ya)\n",
- " bound = np.max(x[y < avg_g])\n",
- " bound_m = ms[3] \n",
- "\n",
- " #print(bound)\n",
- " # fit linear slope\n",
- " if not np.isnan(bound_m):\n",
- " xl = x[(x<bound-20)]\n",
- " yl = y[(x<bound-20)] - offset[pix[0], pix[1], cell, 0]\n",
- " if yl.shape[0] != 0:\n",
- " parms = {'m': bound_m, 'b': np.min(yl)}\n",
- "\n",
- " errors = np.ones(xl.shape)*noise[pix[0], pix[1], cell, 0]\n",
- " fitted = fit_data(lin_fun, xl, yl, errors , parms)\n",
- " yf = lin_fun(xl, fitted['m'], fitted['b'])\n",
- " max_devl = np.max(np.abs((yl-yf)/yl))\n",
- "\n",
- " d3.append({'x': xl,\n",
- " 'y': yf,\n",
- " 'color': 'k',\n",
- " 'linewidth': 1,\n",
- " 'y2': (yf-yl)/errors\n",
- " })\n",
- " # fit hook slope\n",
- " if fit_hook:\n",
- " idx = (x >= bound) & (y > 0) & np.isfinite(x) & np.isfinite(y)\n",
- " xh = x[idx]\n",
- " yh = y[idx] - offset[pix[0], pix[1], cell, 1]\n",
- " if len(yh[yh > 0]) == 0:\n",
- " break\n",
- " parms = {'m': bound_m/10 if bound_m/10>0.3 else 0.5, 'b': np.min(yh[yh > 0]), 'a': np.max(yh), 'c': 5, 'o': bound-1}\n",
- " parms[\"limit_m\"] = [0.3, 2.0]\n",
- " parms[\"limit_c\"] = [1., 1000]\n",
- " errors = np.ones(xh.shape)*noise[pix[0], pix[1], cell, 1]\n",
- " fitted = fit_data(hook_fun, xh, yh, errors, parms)\n",
- " yf = hook_fun(xh, fitted['a'], fitted['c'], fitted['o'], fitted['m'], fitted['b'])\n",
+ " else :\n",
+ " mg_bound_high = len(x)\n",
+ " try:\n",
+ " mg_bound_low = np.max(x[labels[1]]) + 20\n",
+ " except ValueError: #raised if `y` is empty.\n",
+ " mg_bound_low = bound + 100\n",
" \n",
- " max_devh = np.max(np.abs((yh-yf)/yh))\n",
- " #print(fitted)\n",
- " d3.append({'x': xh,\n",
- " 'y': yf,\n",
- " 'color': 'red',\n",
- " 'linewidth': 1,\n",
- " 'y2': (yf-yh)/errors\n",
- " })\n",
+ " except Exception as e:\n",
+ " if \"zero-size array\" in str(e):\n",
+ " pass\n",
+ " else:\n",
+ " print(e)\n",
"\n",
- " x = np.arange(ana.shape[0])\n",
- " y = ana[:,cell, pix[0], pix[1]]\n",
- " \n",
- " vidx = (y > 1000) & np.isfinite(y)\n",
- " x = x[vidx]\n",
- " y = y[vidx]\n",
- " \n",
- " #ms, labels, centers = calc_m_cluster2(x, y, 25, -10, 25)\n",
- " if len(y[labels[0]]) != 0 and len(y[labels[2]]) != 0: \n",
- " threshold = (np.mean(y[labels[0]])+np.mean(y[labels[2]]))/2\n",
+ " return hg_bound_low, hg_bound_high, mg_bound_low, mg_bound_high, bound, bound_m, labels, threshold"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def evaluate_fitting_ROI(modules, res, tpix_range, test_cells, roi):\n",
+ " markers = ['o', '.', 'x', 'v']\n",
+ " colors = ['tab:blue', 'tab:red', 'tab:green', 'k']\n",
+ " test_pixels = []\n",
+ " \n",
+ " fig1, ax1 = plt.subplots(figsize=(9, 5))\n",
+ " ax1.grid(zorder=0, lw=1.5)\n",
+ " ax1.set_ylabel(\"PC pixel signal (ADU)\", fontsize=11)\n",
+ " ax1.set_xlabel('step #', fontsize=11)\n",
+ " \n",
+ " fig2, ax2 = plt.subplots(figsize=(9, 5))\n",
+ " ax2.grid(zorder=0, lw=1.5)\n",
+ " ax2.set_ylabel(\"PC gain signal (ADU)\", fontsize=11)\n",
+ " ax2.set_xlabel('step #', fontsize=11)\n",
+ " \n",
+ " fig3 = plt.figure(figsize=(9, 5))\n",
+ " gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1]) \n",
+ " ax3 = plt.subplot(gs[0])\n",
+ " ax4 = plt.subplot(gs[1])\n",
+ " ax3.grid(zorder=0, lw=1.5)\n",
+ " ax3.set_ylabel('PC signal (keV)', fontsize=11)\n",
+ " ax4.set_ylabel('Relative\\ndeviation', fontsize=11)\n",
+ " ax4.set_xlabel('step #', fontsize=11)\n",
+ " \n",
+ " for i in range(*tpix_range[0]):\n",
+ " for j in range(*tpix_range[1]):\n",
+ " test_pixels.append((j,i))\n",
" \n",
- " for i, lbl in enumerate(labels):\n",
- " \n",
- " d2.append({'x': x[lbl],\n",
- " 'y': y[lbl],\n",
- " 'marker': markers[i],\n",
- " 'color': colors[i],\n",
- " 'lw': None\n",
- " \n",
- " })\n",
- " \n",
- " d2.append({'x': np.array([x[0], x[-1]]),\n",
- " 'y': np.ones(2)*threshold,\n",
- " \n",
- " 'color': 'k',\n",
- " 'lw': 1\n",
+ " tcell = np.array(test_cells)\n",
+ " tcell = tcell[tcell < mem_cells]\n",
+ " if tcell.size == 0:\n",
+ " test_cells = [mem_cells-1]\n",
+ " else:\n",
+ " test_cells = tcell.tolist()\n",
+ " \n",
+ " for mod, r in zip(modules, res):\n",
+ " ana, dig, cellId = r\n",
+ " d = []\n",
+ " d2 = []\n",
+ " d3 = []\n",
+ " offset = offsets[mod]\n",
+ " noise = noises[mod]\n",
+ " for pix in test_pixels:\n",
+ " for cell in test_cells:\n",
+ " color = np.random.rand(3, 1)\n",
+ " x = np.arange(ana.shape[0])\n",
+ " y = ana[:,cell, pix[0], pix[1]]\n",
+ " yd = dig[:,cell, pix[0], pix[1]]\n",
+ " thresh_trans = thresholds[mod][pix[0], pix[1], cell,0]\n",
+ "\n",
+ " vidx = (y > 1000) & np.isfinite(y) & np.isfinite(x)\n",
+ " x = x[vidx]\n",
+ " y = y[vidx]\n",
+ " yd = yd[vidx]\n",
+ "\n",
+ " hg_bound_low, hg_bound_high, mg_bound_low, mg_bound_high, bound, bound_m, labels, threshold = find_params(x, y, yd, thresh_trans)\n",
+ "\n",
+ "\n",
+ " for i, lbl in enumerate(labels):\n",
+ " if i == 0:\n",
+ " gain = 0\n",
+ "\n",
+ " else:\n",
+ " gain = 1\n",
" \n",
- " })\n",
- " \n",
- " #threshold = (np.min(y[x<bound]) + np.max(y[x>=bound]))/2\n",
- " \n",
- " \n",
- " fig = xana.simplePlot(d, y_label=\"PC pixel signal (ADU)\", figsize='2col', aspect=2,\n",
- " x_label=\"step #\")\n",
- " fig.savefig(\"{}/module_{}_pixel_plot.png\".format(out_folder, mod))\n",
- " \n",
- " fig = xana.simplePlot(d2, y_label=\"PC gain signal (ADU)\", figsize='2col', aspect=2,\n",
- " x_label=\"step #\")\n",
- " fig.savefig(\"{}/module_{}_pixel_plot_gain.png\".format(out_folder, mod))\n",
+ " ax1.plot(x[lbl], (y[lbl] - offset[pix[0], pix[1], cell, gain]), ls='None', \n",
+ " marker=markers[i], color=colors[i], alpha=0.3)\n",
+ " ax2.plot(x[lbl], yd[lbl], ls='None', marker=markers[i], color=colors[i], alpha=0.3)\n",
+ " ax2.plot(np.array([x[0], x[-1]]), np.ones(2) * threshold, lw=1, color='k', alpha=0.3)\n",
+ " \n",
+ " # fit linear slope\n",
+ " idx = (x > hg_bound_low) & (x < hg_bound_high)\n",
+ " xl = x[idx]\n",
+ " yl = y[idx] - y[0]\n",
+ "\n",
+ " if yl.shape[0] != 0: \n",
+ " parms = {'m': bound_m, 'b': np.min(yl)}\n",
+ " parms[\"limit_m\"] = [0., 50.]\n",
+ " errors = np.ones(xl.shape) * noise[pix[0], pix[1], cell, 0]\n",
+ " fitted = fit_data(lin_fun, xl, yl, errors , parms)\n",
+ " hg_slope = fitted['m']\n",
+ " pc_high_l = fitted['b']\n",
" \n",
- " fig = xana.simplePlot(d3, secondpanel=True, y_label=\"PC signal (ADU)\", figsize='2col', aspect=2,\n",
- " x_label=\"step #\", y2_label=\"Residuals ($\\sigma$)\", y2_range=(-5,5))\n",
- " fig.savefig(\"{}/module_{}_pixel_plot_fits.png\".format(out_folder, mod))"
+ " yf = lin_fun(xl, hg_slope, pc_high_l)\n",
+ " \n",
+ " ax3.plot(xl, (yl / FF_gain), color='tab:blue', ls='None',alpha=0.5, marker='o')\n",
+ " ax3.plot(xl, (yf / FF_gain), color='navy', lw=1, zorder=9)\n",
+ " ax4.plot(xl, ((yl-yf) / yl), lw=1, color='navy')\n",
+ "\n",
+ " # fit hook slope\n",
+ " try:\n",
+ " if fit_hook:\n",
+ " idx = (x >= mg_bound_low) & (x < mg_bound_high)\n",
+ " xh = x[idx]\n",
+ " yh = y[idx] - offset[pix[0], pix[1], cell, 1]\n",
+ "\n",
+ " parms = {'m': bound_m/10 if bound_m/10>0.3 else 0.5, \n",
+ " 'b': np.min(yh[yh > -2000]), \n",
+ " 'a': np.max(yh), 'c': 5, 'o': bound-1}\n",
+ " parms[\"limit_m\"] = [0., 2.0]\n",
+ " parms[\"limit_c\"] = [1., 1000]\n",
+ "\n",
+ " errors = np.ones(xh.shape) * noise[pix[0], pix[1], cell, 1]\n",
+ " fitted = fit_data(hook_fun, xh, yh, errors, parms)\n",
+ " mg_slope = fitted['m']\n",
+ " pc_med_l = fitted['b']\n",
+ " \n",
+ " slope_ratio = hg_slope / mg_slope\n",
+ " md_additional_offset = pc_high_l - pc_med_l * slope_ratio\n",
+ "\n",
+ " yf = lin_fun(xh, mg_slope * slope_ratio, pc_med_l)\n",
+ " corr_yh = yh * slope_ratio + md_additional_offset\n",
+ " max_devh = np.median(np.abs((corr_yh - yf) / corr_yh))\n",
+ "\n",
+ " else:\n",
+ " idx = (x >= mg_bound_low) & (x < mg_bound_high)\n",
+ " xh = x[idx]\n",
+ " yh = y[idx] - offset[pix[0], pix[1], cell, 1]\n",
+ "\n",
+ " if len(yh[yh > -2000]) == 0:\n",
+ " break\n",
+ " parms = {'m': bound_m/10 if bound_m/10>0.3 else 0.5, \n",
+ " 'b': np.min(yh[yh > -2000]), \n",
+ " }\n",
+ " parms[\"limit_m\"] = [0., 2.0]\n",
+ " errors = np.ones(xh.shape) * noise[pix[0], pix[1], cell, 1]\n",
+ " fitted = fit_data(lin_fun, xh, yh, errors, parms)\n",
+ " mg_slope = fitted['m']\n",
+ " pc_med_l = fitted['b']\n",
+ " slope_ratio = hg_slope / mg_slope\n",
+ " md_additional_offset = pc_high_l - pc_med_l * slope_ratio\n",
+ "\n",
+ " yf = lin_fun(xh, mg_slope * slope_ratio, pc_med_l)\n",
+ " corr_yh = yh * slope_ratio + md_additional_offset\n",
+ " max_devh = np.median(np.abs((corr_yh - yf) / corr_yh))\n",
+ " \n",
+ " ax3.plot(xh, corr_yh/FF_gain, color='tab:green', ls='None', alpha=0.3, marker='x')\n",
+ " ax3.plot(xh, yf/FF_gain, color='green', lw=1, zorder=10)\n",
+ " ax4.plot(xh, ((corr_yh-yf)/corr_yh), lw=1, color='green')\n",
+ "\n",
+ " except Exception as e:\n",
+ " if \"zero-size array\" in str(e):\n",
+ " pass\n",
+ " else:\n",
+ " print(e)\n",
+ "\n",
+ " fig1.show()\n",
+ " fig2.show()\n",
+ " fig3.show()\n",
+ "\n",
+ " fig1.savefig(\"{}/module_{}_{}_pixel_plot.png\".format(out_folder, mod, roi))\n",
+ " fig2.savefig(\"{}/module_{}_{}_pixel_plot_gain.png\".format(out_folder, mod, roi))\n",
+ " fig3.savefig(\"{}/module_{}_{}_pixel_plot_fits.png\".format(out_folder, mod, roi))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## ROI 1"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": true
- },
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "tpix_range1 = [(250,254), (60,63)]\n",
+ "test_cells = [4, 38]\n",
+ "roi = 'ROI1'\n",
+ "evaluate_fitting_ROI(modules, res, tpix_range1, test_cells, roi)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## ROI 2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
"outputs": [],
"source": [
- "test_pixels = []\n",
"tpix_range2 = [(96,128), (32,64)]\n",
- "for i in range(*tpix_range2[0]):\n",
- " for j in range(*tpix_range2[1]):\n",
- " test_pixels.append((j,i))\n",
- "\n",
- "for mod, r in zip(modules, res):\n",
- " dig, ana, cellId = r\n",
- " d = []\n",
- " d2 = []\n",
- " d3 = []\n",
- " offset = offsets[mod]\n",
- " noise = noises[mod]\n",
- " for pix in test_pixels:\n",
- " for cell in test_cells:\n",
- " color = np.random.rand(3,1)\n",
- "\n",
- " x = np.arange(dig.shape[0])\n",
- " y = dig[:,cell, pix[0], pix[1]]\n",
- "\n",
- " vidx = (y > 1000) & np.isfinite(y)\n",
- " x = x[vidx]\n",
- " y = y[vidx]\n",
- "\n",
- " ms, labels, centers = calc_m_cluster2(x, y)\n",
- " bound = None\n",
- " bound_m = None\n",
- " markers = ['o','.','x','v']\n",
- " colors = ['b', 'r', 'g', 'k']\n",
- " for i, lbl in enumerate(labels):\n",
- " if i == 0:\n",
- " gain = 0\n",
- " else:\n",
- " gain = 1\n",
- " d.append({'x': x[lbl],\n",
- " 'y': y[lbl] - offset[pix[0], pix[1], cell, gain],\n",
- " 'marker': markers[i],\n",
- " 'color': colors[i],\n",
- " 'linewidth': 0\n",
- " })\n",
- " #if ms[i] < 0: # slope separating two regions\n",
- " # bound = np.min(x[lbl])\n",
- " # bound_m = ms[i]\n",
- " if len(x[labels[1]]):\n",
- " bound = np.min(x[labels[1]])\n",
- " bound_m = ms[1]\n",
- "\n",
- " # fit linear slope\n",
- " idx = (x >= bound) & (y > 0) & np.isfinite(x) & np.isfinite(y)\n",
- " xl = x[(x<bound-20)]\n",
- " yl = y[(x<bound-20)] - offset[pix[0], pix[1], cell, 0]\n",
- " errors = np.ones(xl.shape)*noise[pix[0], pix[1], cell, 0]\n",
- " if yl.shape[0] != 0:\n",
- " parms = {'m': bound_m, 'b': np.min(yl)}\n",
- " fitted = fit_data(lin_fun, xl, yl, errors, parms)\n",
- "\n",
- " yf = lin_fun(xl, fitted['m'], fitted['b'])\n",
- " max_devl = np.max(np.abs((yl-yf)/yl))\n",
- "\n",
- " xtt = np.arange(ana.shape[0])\n",
- " ytt = ana[:,cell, pix[0], pix[1]]\n",
- "\n",
- " vidx = (ytt > 1000) & np.isfinite(ytt)\n",
- " xtt = xtt[vidx]\n",
- " ytt = ytt[vidx]\n",
- "\n",
- " #ms, labels, centers = calc_m_cluster2(x, y, 25, -10, 25)\n",
- " if len(y[labels[0]]) != 0 and len(y[labels[2]]) != 0: \n",
- " threshold = (np.mean(ytt[labels[0]])+np.mean(ytt[labels[2]]))/2\n",
- "\n",
- " if threshold > 10000 or threshold < 4000:\n",
- " d3.append({\n",
- " 'x': xl,\n",
- " 'y': yf,\n",
- " 'color': 'k',\n",
- " 'linewidth': 1,\n",
- " 'y2': (yf-yl)/errors\n",
- " })\n",
- "\n",
- " if bound is None:\n",
- " ya = ana[:,cell, pix[0], pix[1]][vidx]\n",
- " msa, labels, centers = calc_m_cluster2(x, ya, 25, -10, 25)\n",
- " if np.count_nonzero(labels[0]) > 0:\n",
- " bound = np.min(x[labels[0]])\n",
- " bound_m = ms[3]\n",
- " else:\n",
- " avg_g = np.nanmean(ya)\n",
- " bound = np.max(x[y < avg_g])\n",
- " bound_m = ms[3] \n",
- "\n",
- " # fit hook slope\n",
- " try:\n",
- " if fit_hook and len(yh[yh > 0]) !=0:\n",
- " idx = (x >= bound) & (y > 0) & np.isfinite(x) & np.isfinite(y)\n",
- " xh = x[idx]\n",
- " yh = y[idx] - offset[pix[0], pix[1], cell, 1]\n",
- " errors = np.ones(xh.shape)*noise[pix[0], pix[1], cell, 1]\n",
- " parms = {\n",
- " 'm': np.abs(bound_m/10),\n",
- " 'b': np.min(yh[yh > 0]), \n",
- " 'a': np.max(yh),\n",
- " 'c': 5.,\n",
- " 'o': bound-1\n",
- " }\n",
- " parms[\"limit_m\"] = [0.3, 2.0]\n",
- " parms[\"limit_c\"] = [1., 1000]\n",
- " fitted = fit_data(hook_fun, xh, yh, errors, parms)\n",
- " yf = hook_fun(xh, fitted['a'], fitted['c'], fitted['o'], fitted['m'], fitted['b'])\n",
- " max_devh = np.max(np.abs((yh-yf)/yh))\n",
- " #print(fitted)\n",
- " if threshold > 10000 or threshold < 4000 or fitted['m'] < 0.2:\n",
- " d3.append({\n",
- " 'x': xh,\n",
- " 'y': yf,\n",
- " 'color': 'red',\n",
- " 'linewidth': 1,\n",
- " 'y2': (yf-yh)/errors\n",
- " })\n",
- " except Exception as e:\n",
- " if \"zero-size array\" in str(e):\n",
- " pass\n",
- " else:\n",
- " print(e)\n",
- "\n",
- " if threshold > 10000 or threshold < 4000:\n",
- " for i, lbl in enumerate(labels):\n",
- " d2.append({\n",
- " 'x': xtt[lbl],\n",
- " 'y': ytt[lbl],\n",
- " 'marker': markers[i],\n",
- " 'color': colors[i],\n",
- " 'lw': None\n",
- " })\n",
- "\n",
- " d2.append({'x': np.array([xtt[0], xtt[-1]]),\n",
- " 'y': np.ones(2)*threshold,\n",
- " 'color': 'k',\n",
- " 'lw': 1\n",
- " })\n",
- "\n",
- " #threshold = (np.min(y[x<bound]) + np.max(y[x>=bound]))/2\n",
- " fig = xana.simplePlot(d, y_label=\"PC pixel signal (ADU)\", figsize='2col', aspect=2,\n",
- " x_label=\"step #\")\n",
- " fig.savefig(\"{}/module_{}_pixel_plot_fail.png\".format(out_folder, mod))\n",
- "\n",
- " fig = xana.simplePlot(d2, y_label=\"PC gain signal (ADU)\", figsize='2col', aspect=2,\n",
- " x_label=\"step #\")\n",
- " fig.savefig(\"{}/module_{}_pixel_plot_gain_fail.png\".format(out_folder, mod))\n",
- "\n",
- " fig = xana.simplePlot(d3, secondpanel=True, y_label=\"PC signal (ADU)\", figsize='2col', aspect=2,\n",
- " x_label=\"step #\", y2_label=\"Residuals ($\\sigma$)\", y2_range=(-5,5))\n",
- " fig.savefig(\"{}/module_{}_pixel_plot_fits_fail.png\".format(out_folder, mod))\n"
+ "roi2 = 'ROI2'\n",
+ "evaluate_fitting_ROI(modules, res, tpix_range2, test_cells, roi2)"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-27T23:34:23.912223Z",
- "start_time": "2019-07-27T23:10:43.719Z"
- },
- "scrolled": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Here we perform the calculations in column-parallel for all modules\n",
- "def calibrate_single_row(cells, fit_hook, inp):\n",
+ "def calibrate_single_row(cells, fit_hook, ranges, inp):\n",
+ " \n",
+ " hg_range = ranges[0]\n",
+ " mg_range = ranges[1]\n",
" \n",
" import numpy as np\n",
" from iminuit import Minuit\n",
" from iminuit.util import describe, make_func_code\n",
" from sklearn.cluster import KMeans\n",
" \n",
- " yrd, yra, offset, noise = inp\n",
+ " yra, yrd, offset, noise, thresh_trans = inp\n",
"\n",
" def rolling_window(a, window):\n",
" shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)\n",
@@ -1088,7 +960,7 @@
" for i in range(scan_range):\n",
" xdiffs = x - np.roll(x, i+1)\n",
" ydiffs = y - np.roll(y, i+1)\n",
- " m = ydiffs/xdiffs\n",
+ " m = ydiffs / xdiffs\n",
" ms[:,i] = m\n",
" m = np.mean(ms, axis=1)\n",
" m[scan_range//2:-scan_range//2+1] = np.mean(rolling_window(m, scan_range),-1)\n",
@@ -1171,102 +1043,153 @@
" return a*np.exp(-(x-o)/c)+m*x+b\n",
" \n",
" # linear slope\n",
- " ml = np.zeros(yrd.shape[1:])\n",
- " bl = np.zeros(yrd.shape[1:])\n",
- " devl = np.zeros(yrd.shape[1:])\n",
- " ml[...] = np.nan\n",
- " bl[...] = np.nan\n",
- " devl[...] = np.nan\n",
+ " ml = np.full(yra.shape[1:], np.nan)\n",
+ " bl = np.full(yra.shape[1:], np.nan)\n",
+ " devl = np.full(yra.shape[1:], np.nan)\n",
" \n",
" #hook function\n",
- " mh = np.zeros(yrd.shape[1:])\n",
- " bh = np.zeros(yrd.shape[1:])\n",
- " ch = np.zeros(yrd.shape[1:])\n",
- " oh = np.zeros(yrd.shape[1:])\n",
- " ah = np.zeros(yrd.shape[1:])\n",
- " devh = np.zeros(yrd.shape[1:])\n",
- " dhm = np.zeros(yrd.shape[1:])\n",
- " mh[...] = np.nan\n",
- " bh[...] = np.nan\n",
- " ch[...] = np.nan\n",
- " oh[...] = np.nan\n",
- " ah[...] = np.nan\n",
- " devh[...] = np.nan\n",
- " dhm[...] = np.nan\n",
+ " mh = np.full(yra.shape[1:], np.nan)\n",
+ " bh = np.full(yra.shape[1:], np.nan)\n",
+ " ch = np.full(yra.shape[1:], np.nan)\n",
+ " oh = np.full(yra.shape[1:], np.nan)\n",
+ " ah = np.full(yra.shape[1:], np.nan)\n",
+ " devh = np.full(yra.shape[1:], np.nan)\n",
+ " dhm = np.full(yra.shape[1:], np.nan)\n",
" \n",
" # threshold\n",
- " thresh = np.zeros(list(yrd.shape[1:])+[3,])\n",
- " thresh[...] = np.nan\n",
+ " thresh = np.full(list(yrd.shape[1:])+[3,], np.nan)\n",
" failures = []\n",
" \n",
- " for col in range(yrd.shape[-1]):\n",
+ " for col in range(yra.shape[-1]):\n",
" try:\n",
- " y = yrd[:,col]\n",
+ " y = yra[:,col]\n",
" x = np.arange(y.shape[0])\n",
+ " yd = yrd[:,col] \n",
"\n",
- " vidx = (y > 1000) & np.isfinite(y)\n",
+ " vidx = (y > 1000) & np.isfinite(y) & np.isfinite(x)\n",
" x = x[vidx]\n",
" y = y[vidx]\n",
+ " yd = yd[vidx]\n",
"\n",
" ms, labels, centers = calc_m_cluster2(x, y)\n",
"\n",
- " bound = np.min(x[labels[1]])\n",
+ " bound = np.min(x[labels[1]]) \n",
" bound_m = ms[1]\n",
+ " threshold = (np.mean(yd[labels[0]])+np.mean(yd[labels[2]])) / 2\n",
+ " thresh[col,0] = threshold\n",
+ " thresh[col,1] = np.mean(y[labels[0]])\n",
+ " thresh[col,2] = np.mean(y[labels[2]])\n",
+ " \n",
+ " # fit linear slope \n",
+ " if hg_range[1] > 0 :\n",
+ " hg_bound_high = hg_range[1]\n",
+ " hg_bound_low = hg_range[0]\n",
+ " else :\n",
+ " hg_bound_high = bound - 20\n",
+ " hg_bound_low = 0\n",
"\n",
- " # fit linear slope\n",
- " xl = x[x<bound-20]\n",
- " yl = y[x<bound-20] - offset[col, 0]\n",
- " errors = np.ones(xl.shape)*noise[col, 0]\n",
- " if yl.shape[0] != 0:\n",
+ " if fit_hook:\n",
+ " mg_bound_high = len(x)\n",
+ " try:\n",
+ " mg_bound_low = np.max(x[labels[1]]) + 20\n",
+ " except ValueError: #raised if `y` is empty.\n",
+ " mg_bound_low = bound + 100 \n",
+ " else :\n",
+ " if mg_range[0] > 0 :\n",
+ " mg_bound_low = mg_range[0]\n",
+ " mg_bound_high = mg_range[1]\n",
+ "\n",
+ " else :\n",
+ " mg_bound_high = len(x)\n",
+ " try:\n",
+ " mg_bound_low = np.max(x[labels[1]]) + 20\n",
+ " except ValueError: #raised if `y` is empty.\n",
+ " mg_bound_low = bound + 100 \n",
+ " \n",
+ " idx = (x > hg_bound_low) & (x < hg_bound_high)\n",
+ " xl = x[idx]\n",
+ " yl = y[idx] - y[0] # instead of offset subtraction is better to subtract first pc value here\n",
+ " errors = np.ones(xl.shape) * noise[col, 0]\n",
+ " if yl.shape[0] != 0: \n",
" parms = {'m': bound_m, 'b': np.min(yl)}\n",
+ " parms[\"limit_m\"] = [0., 50.]\n",
" fitted = fit_data(lin_fun, xl, yl, errors, parms)\n",
- " yf = lin_fun(xl, fitted['m'], fitted['b'])\n",
- " max_devl = np.median(np.abs((yl-yf)/yl))\n",
- " ml[col] = fitted['m']\n",
- " bl[col] = fitted['b']\n",
+ " hg_slope = fitted['m']\n",
+ " pc_high_l = fitted['b']\n",
+ " yf = lin_fun(xl, hg_slope, pc_high_l)\n",
+ " max_devl = np.median(np.abs((yl-yf) / yl))\n",
+ " ml[col] = hg_slope #fitted['m']\n",
+ " bl[col] = pc_high_l #fitted['b']\n",
" devl[col] = max_devl\n",
- " #if np.any(labels[0]) and np.any(labels[2]):\n",
- " #dhm[col] = y[labels[0]].max()-y[labels[2]].min()\n",
- " dhml = lin_fun(bound, fitted['m'], fitted['b'])\n",
+ " dhml = lin_fun(bound, hg_slope, pc_high_l)\n",
+ " \n",
" # fit hook slope\n",
" if fit_hook:\n",
- " idx = (x >= bound) & (y > 0) & np.isfinite(x) & np.isfinite(y)\n",
+ " idx = (x >= mg_bound_low) & (x < mg_bound_high)\n",
" xh = x[idx]\n",
" yh = y[idx] - offset[col, 1]\n",
- " errors = np.ones(xh.shape)*noise[col, 1]\n",
- " parms = {'m': bound_m/10 if bound_m/10 > 0.3 else 0.5, 'b': np.min(yh[yh > 0]), 'a': np.max(yh), 'c': 5., 'o': bound-1}\n",
- " parms[\"limit_m\"] = [0.3, 2.0]\n",
+ " errors = np.ones(xh.shape) * noise[col, 1]\n",
+ " parms = {'m': bound_m/10 if bound_m/10 > 0.3 else 0.5, 'b': np.min(yh[yh > -2000]), 'a': np.max(yh), 'c': 5., 'o': bound-1}\n",
+ " parms[\"limit_m\"] = [0., 2.0]\n",
" parms[\"limit_c\"] = [1., 1000]\n",
" fitted = fit_data(hook_fun, xh, yh, errors, parms)\n",
- " yf = hook_fun(xh, fitted['a'], fitted['c'], fitted['o'], fitted['m'], fitted['b'])\n",
- " max_devh = np.median(np.abs((yh-yf)/yh)) \n",
- "\n",
- " mh[col] = fitted['m']\n",
- " bh[col] = fitted['b']\n",
+ " mg_slope = fitted['m']\n",
+ " pc_med_l = fitted['b']\n",
+ " slope_ratio = hg_slope / mg_slope\n",
+ " md_additional_offset = pc_high_l - pc_med_l * slope_ratio\n",
+ " yf = lin_fun(xh, mg_slope * slope_ratio, pc_med_l)\n",
+ " corr_yh = yh * slope_ratio + md_additional_offset\n",
+ " max_devh = np.median(np.abs((corr_yh - yf) / corr_yh))\n",
+ "\n",
+ " mh[col] = mg_slope # fitted['m']\n",
+ " bh[col] = pc_med_l # fitted['b']\n",
" ah[col] = fitted['a']\n",
" oh[col] = fitted['o']\n",
" ch[col] = fitted['c']\n",
" devh[col] = max_devh\n",
- " dhm[col] = bound #(dhml) - lin_fun(bound, fitted['m'], fitted['b'])\n",
- "\n",
- " y = yra[:,col]\n",
- " x = np.arange(y.shape[0])\n",
- "\n",
- " vidx = (y > 1000) & np.isfinite(y)\n",
- " x = x[vidx]\n",
- " y = y[vidx]\n",
+ " dhm[col] = bound \n",
+ " else : \n",
+ " idx = (x >= mg_bound_low) & (x < mg_bound_high)\n",
+ " xh = x[idx]\n",
+ " yh = y[idx] - offset[col, 1] \n",
+ " errors = np.ones(xh.shape)*noise[col, 1]\n",
+ " parms = {'m': bound_m/10 if bound_m/10 > 0.3 else 0.5, \n",
+ " 'b': np.min(yh[yh > -2000]),\n",
+ " }\n",
+ " parms[\"limit_m\"] = [0., 2.0]\n",
+ " fitted = fit_data(lin_fun, xh, yh, errors, parms)\n",
+ " \n",
+ " mg_slope = fitted['m']\n",
+ " pc_med_l = fitted['b']\n",
+ " slope_ratio = hg_slope / mg_slope\n",
+ " md_additional_offset = pc_high_l - pc_med_l * slope_ratio\n",
+ "\n",
+ " yf = lin_fun(xh, mg_slope * slope_ratio, pc_med_l)\n",
+ " corr_yh = yh * slope_ratio + md_additional_offset\n",
+ " max_devh = np.median(np.abs((corr_yh - yf) / corr_yh)) \n",
+ "\n",
+ " mh[col] = mg_slope # fitted['m']\n",
+ " bh[col] = pc_med_l # fitted['b']\n",
+ " ah[col] = 0\n",
+ " oh[col] = 1\n",
+ " ch[col] = 1\n",
+ " devh[col] = max_devh\n",
+ " dhm[col] = bound \n",
"\n",
- " threshold = (np.mean(y[labels[0]])+np.mean(y[labels[2]]))/2\n",
- " thresh[col,0] = threshold\n",
- " thresh[col,1] = np.mean(y[labels[0]])\n",
- " thresh[col,2] = np.mean(y[labels[2]])\n",
" except Exception as e:\n",
" print(e)\n",
" failures.append((col, str(e)))\n",
" del yrd\n",
" del yra \n",
- " return thresh, (ml, bl, devl), (mh, bh, ah, oh, ch, devh), failures, dhm\n",
- "\n",
+ " return thresh, (ml, bl, devl), (mh, bh, ah, oh, ch, devh), failures, dhm\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"start = datetime.now()\n",
"fres = {}\n",
"failures = []\n",
@@ -1275,6 +1198,7 @@
" noise = noises[i]\n",
" qm = module_index_to_qm(i)\n",
" dig, ana, cellId = r\n",
+ " thresh_trans = thresholds[i][..., 0]\n",
" \n",
" \n",
" # linear slope\n",
@@ -1298,12 +1222,10 @@
" for cell in range(dig.shape[1]):\n",
" inp = []\n",
" for j in range(dig.shape[2]):\n",
- " inp.append((dig[:,cell,j,:], ana[:,cell,j,:], offset[j,:,cell,:], noise[j,:,cell,:]))\n",
+ " inp.append((dig[:,cell,j,:], ana[:,cell,j,:], offset[j,:,cell,:], noise[j,:,cell,:], thresh_trans[j,:,cell]))\n",
"\n",
- " p = partial(calibrate_single_row, cells, fit_hook)\n",
- " #print(\"Running {} tasks in parallel\".format(len(inp)))\n",
+ " p = partial(calibrate_single_row, cells, fit_hook, [hg_range, mg_range])\n",
" frs = view.map_sync(p, inp)\n",
- " #frs = list(map(p, inp))\n",
"\n",
" for j, fr in enumerate(frs):\n",
" threshr, lin, hook, fails, dhm = fr\n",
@@ -1311,20 +1233,20 @@
" mhr, bhr, ahr, ohr, chro, devhr = hook\n",
" failures.append(fails)\n",
"\n",
- " ml[cell,j,:] = mlr\n",
- " bl[cell,j,:] = blr\n",
- " devl[cell,j,:] = devlr\n",
+ " ml[cell, j, :] = mlr\n",
+ " bl[cell, j, :] = blr\n",
+ " devl[cell, j, :] = devlr\n",
"\n",
- " mh[cell,j,:] = mhr\n",
- " bh[cell,j,:] = bhr\n",
- " oh[cell,j,:] = ohr\n",
+ " mh[cell, j, :] = mhr\n",
+ " bh[cell, j, :] = bhr\n",
+ " oh[cell, j, :] = ohr\n",
" ch[cell,j,:] = chro\n",
- " ah[cell,j,:] = ahr \n",
- " devh[cell,j,:] = devhr\n",
- " dhma[cell,j,:] = dhm\n",
+ " ah[cell, j, :] = ahr \n",
+ " devh[cell, j, :] = devhr\n",
+ " dhma[cell, j, :] = dhm\n",
"\n",
- " thresh[cell,j,...] = threshr[...,0]\n",
- " thresh_bounds[cell,j,...] = threshr[...,1:]\n",
+ " thresh[cell, j, ...] = threshr[..., 0]\n",
+ " thresh_bounds[cell, j, ...] = threshr[..., 1:]\n",
" \n",
" fres[qm] = {'ml': ml,\n",
" 'bl': bl,\n",
@@ -1332,15 +1254,17 @@
" 'tresh': thresh,\n",
" 'tresh_bounds': thresh_bounds,\n",
" 'dhm': dhma}\n",
- " if fit_hook:\n",
- " fres[qm].update({\n",
- " 'mh': mh,\n",
- " 'bh': bh,\n",
- " 'oh': oh,\n",
- " 'ch': ch,\n",
- " 'ah': ah,\n",
- " 'devh': devh,\n",
- " })"
+ " fres[qm].update({\n",
+ " 'mh': mh,\n",
+ " 'bh': bh,\n",
+ " 'oh': oh,\n",
+ " 'ch': ch,\n",
+ " 'ah': ah,\n",
+ " 'devh': devh,\n",
+ " })\n",
+ "\n",
+ "end = datetime.now() - start\n",
+ "print('Duration of fitting: ',end)"
]
},
{
@@ -1364,12 +1288,137 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-15T10:22:45.672082Z",
- "start_time": "2019-07-15T10:22:45.662437Z"
- }
- },
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from collections import OrderedDict\n",
+ "from scipy.stats import median_abs_deviation as mad\n",
+ "\n",
+ "bad_pixels = OrderedDict()\n",
+ "for qm, data in fres.items(): \n",
+ " mask = np.zeros(data['ml'].shape, np.uint32)\n",
+ " mask[(data['tresh'][...,0] < 50) | (data['tresh'][...,0] > 8500)] |= BadPixels.CI_GAIN_OF_OF_THRESHOLD.value\n",
+ " mask[(data['devl'] == 0)] |= BadPixels.CI_LINEAR_DEVIATION.value\n",
+ " mask[(np.abs(data['devl']) > 0.5)] |= BadPixels.CI_LINEAR_DEVIATION.value\n",
+ " mask[(~np.isfinite(data['devl']))] |= BadPixels.CI_EVAL_ERROR.value\n",
+ " bad_pixels[qm] = mask\n",
+ " \n",
+ " # high gain slope from pulse capacitor data\n",
+ " pc_high_m = data['ml']\n",
+ " pc_high_b = data['bl']\n",
+ " # medium gain slope from pulse capacitor data\n",
+ " pc_med_m = data['mh']\n",
+ " pc_med_b = data['bh']\n",
+ " \n",
+ " fshape = pc_high_m.shape\n",
+ " \n",
+ " # calculate median for slopes\n",
+ " pc_high_med = np.nanmedian(pc_high_m, axis=(1, 2))\n",
+ " pc_med_med = np.nanmedian(pc_med_m, axis=(1, 2))\n",
+ " # calculate median for intercepts:\n",
+ " pc_high_b_med = np.nanmedian(pc_high_b, axis=(1, 2))\n",
+ " pc_med_b_med = np.nanmedian(pc_med_b, axis=(1, 2)) \n",
+ " \n",
+ " # mad can only iterate on 1 axis\n",
+ " tshape = (fshape[0],fshape[1]*fshape[2])\n",
+ " \n",
+ " # calculate MAD for slopes\n",
+ " pc_high_rms = mad(pc_high_m.reshape(tshape), axis=1, scale='normal', nan_policy='omit')\n",
+ " pc_med_rms = mad(pc_med_m.reshape(tshape), axis=1, scale='normal', nan_policy='omit')\n",
+ " # calculate MAD for intercepts:\n",
+ " pc_high_b_rms = mad(pc_high_b.reshape(tshape), axis=1, scale='normal', nan_policy='omit')\n",
+ " pc_med_b_rms = mad(pc_med_b.reshape(tshape), axis=1, scale='normal', nan_policy='omit')\n",
+ "\n",
+ " # expand the arrays to match the size of the originals\n",
+ " pc_high_med = np.repeat(pc_high_med, fshape[1]*fshape[2]).reshape(fshape)\n",
+ " pc_med_med = np.repeat(pc_med_med, fshape[1]*fshape[2]).reshape(fshape)\n",
+ " pc_high_b_med = np.repeat(pc_high_b_med, fshape[1]*fshape[2]).reshape(fshape)\n",
+ " pc_med_b_med = np.repeat(pc_med_b_med, fshape[1]*fshape[2]).reshape(fshape)\n",
+ " pc_high_rms = np.repeat(pc_high_rms, fshape[1]*fshape[2]).reshape(fshape)\n",
+ " pc_med_rms = np.repeat(pc_med_rms, fshape[1]*fshape[2]).reshape(fshape)\n",
+ " pc_high_b_rms = np.repeat(pc_high_b_rms, fshape[1]*fshape[2]).reshape(fshape)\n",
+ " pc_med_b_rms = np.repeat(pc_med_b_rms, fshape[1]*fshape[2]).reshape(fshape)\n",
+ " \n",
+ " # sanitize PC data\n",
+ " # replace vals outside range and nans with median.\n",
+ " \n",
+ " bad_fits_high = (np.abs((pc_high_m - pc_high_med)/pc_high_rms) > sigma_dev_cut) |\\\n",
+ " (np.abs((pc_high_b - pc_high_b_med)/pc_high_b_rms) > sigma_dev_cut) |\\\n",
+ " (~np.isfinite(pc_high_m)) | (~np.isfinite(pc_high_b))\n",
+ "\n",
+ " bad_fits_med = (np.abs((pc_med_m - pc_med_med)/pc_med_rms) > sigma_dev_cut) |\\\n",
+ " (np.abs((pc_med_b - pc_med_b_med)/pc_med_b_rms) > sigma_dev_cut) |\\\n",
+ " (~np.isfinite(pc_med_m)) | (~np.isfinite(pc_med_b))\n",
+ " \n",
+ " def plot_cuts(a,sel,name,units, ax = None) :\n",
+ " stats_cut = {}\n",
+ " stats = {}\n",
+ " if ax is None :\n",
+ " fig = plt.figure(figsize=(5,5))\n",
+ " ax = fig.add_subplot(111)\n",
+ " stats_cut[\"Mean\"] = np.nanmean(a[sel])\n",
+ " stats_cut[\"STD\"] = np.nanstd(a[sel])\n",
+ " stats_cut[\"Median\"] = np.nanmedian(a[sel])\n",
+ " stats_cut[\"Min\"] = np.nanmin(a[sel])\n",
+ " stats_cut[\"Max\"] = np.nanmax(a[sel])\n",
+ " \n",
+ " stats[\"Mean\"] = np.nanmean(a)\n",
+ " stats[\"STD\"] = np.nanstd(a)\n",
+ " stats[\"Median\"] = np.nanmedian(a)\n",
+ " stats[\"Min\"] = np.nanmin(a)\n",
+ " stats[\"Max\"] = np.nanmax(a)\n",
+ " \n",
+ " m=np.nanmean(a)\n",
+ " s=np.nanstd(a)\n",
+ " bins = np.linspace(m-10*s,m+10*s,100)\n",
+ " ax.hist(a.ravel(), \n",
+ " bins = bins,\n",
+ " color='tab:red',\n",
+ " label='all fits',\n",
+ " )\n",
+ " ax.hist(a[sel].ravel(), \n",
+ " bins = bins,\n",
+ " color='tab:green',\n",
+ " label='good fits'\n",
+ " )\n",
+ " ax.grid(zorder=0)\n",
+ " ax.set_xlabel(units)\n",
+ " ax.set_yscale('log')\n",
+ " ax.set_title(name)\n",
+ " ax.legend()\n",
+ " def statistic(stat, colour, shift):\n",
+ " textstr = \"\"\n",
+ " for key in stat:\n",
+ " try:\n",
+ " textstr += '{0: <6}: {1: 7.2f}\\n'.format(key, stat[key])\n",
+ " except:\n",
+ " pass\n",
+ " props = dict(boxstyle='round', alpha=0.65, color=colour,)\n",
+ " ax.text(0.05, 0.95-shift, textstr, transform=ax.transAxes, fontsize=10,\n",
+ " family='monospace', weight='book', stretch='expanded', \n",
+ " verticalalignment='top', bbox=props, zorder=2)\n",
+ " statistic(stats_cut, 'tab:green', 0)\n",
+ " statistic(stats, 'tab:red', 0.25)\n",
+ " \n",
+ " fig = plt.figure(figsize=(15,15))\n",
+ " plot_cuts(pc_high_m,~(bad_fits_high | (mask>0)),'pc_high_m',\"ADU/DAC\",fig.add_subplot(221))\n",
+ " plot_cuts(pc_high_b,~(bad_fits_high | (mask>0)),'pc_high_b',\"ADU\",fig.add_subplot(222))\n",
+ " plot_cuts(pc_med_m,~(bad_fits_med | (mask>0)),'pc_med_m',\"ADU/DAC\",fig.add_subplot(223))\n",
+ " plot_cuts(pc_med_b,~(bad_fits_med | (mask>0)),'pc_med_b',\"ADU\",fig.add_subplot(224))\n",
+ " \n",
+ " pc_high_m[bad_fits_high] = pc_high_med[bad_fits_high]\n",
+ " pc_high_b[bad_fits_high] = pc_high_b_med[bad_fits_high]\n",
+ " pc_med_m[bad_fits_med] = pc_med_med[bad_fits_med]\n",
+ " pc_med_b[bad_fits_med] = pc_med_b_med[bad_fits_med]\n",
+ " \n",
+ " mask[bad_fits_high] |= BadPixels.CI_EVAL_ERROR.value\n",
+ " mask[bad_fits_med] |= BadPixels.CI_EVAL_ERROR.value\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
"outputs": [],
"source": [
"def slope_dict_to_arr(d):\n",
@@ -1384,8 +1433,8 @@
" \"ah\": 7,\n",
" \"devh\": 8,\n",
" \"tresh\": 9,\n",
- " \n",
" }\n",
+ "\n",
" arr = np.zeros([11]+list(d[\"ml\"].shape), np.float32)\n",
" for key, item in d.items():\n",
" if key not in key_to_index:\n",
@@ -1398,35 +1447,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-15T10:22:46.071868Z",
- "start_time": "2019-07-15T10:22:45.675118Z"
- }
- },
- "outputs": [],
- "source": [
- "from collections import OrderedDict\n",
- "\n",
- "bad_pixels = OrderedDict()\n",
- "for qm, data in fres.items(): \n",
- " mask = np.zeros(data['ml'].shape, np.uint32)\n",
- " mask[(data['tresh'][...,0] < 50) | (data['tresh'][...,0] > 8500)] |= BadPixels.CI_GAIN_OF_OF_THRESHOLD.value\n",
- " mask[(data['devl'] == 0)] |= BadPixels.CI_LINEAR_DEVIATION.value\n",
- " mask[(np.abs(data['devl']) > 0.5)] |= BadPixels.CI_LINEAR_DEVIATION.value\n",
- " mask[(~np.isfinite(data['devl']))] |= BadPixels.CI_EVAL_ERROR.value\n",
- " bad_pixels[qm] = mask"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-15T10:22:46.981022Z",
- "start_time": "2019-07-15T10:22:46.075013Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"if local_output:\n",
@@ -1435,8 +1456,7 @@
" for qm, r in fres.items(): \n",
" for key, item in r.items():\n",
" store_file[\"/{}/{}/0/data\".format(qm, key)] = item\n",
- " #arr = slope_dict_to_arr(r)\n",
- " #store_file[\"/{}/SlopesPC/0/data\".format(qm)] = arr\n",
+ " \n",
" store_file[\"/{}/{}/0/data\".format(qm, \"BadPixelsPC\")] = bad_pixels[qm]\n",
" store_file.close()"
]
@@ -1457,19 +1477,17 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-11T15:52:38.117526Z",
- "start_time": "2019-07-11T15:52:38.109271Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"md = None\n",
"\n",
"# set the operating condition\n",
- "condition = Conditions.Dark.AGIPD(memory_cells=maxcells, bias_voltage=bias_voltage,\n",
- " acquisition_rate=acq_rate, gain_setting=gain_setting)\n",
+ "condition = Conditions.Dark.AGIPD(memory_cells=mem_cells, \n",
+ " bias_voltage=bias_voltage,\n",
+ " acquisition_rate=acq_rate, \n",
+ " gain_setting=gain_setting,\n",
+ " integration_time=integration_time)\n",
"\n",
"db_modules = get_pdu_from_db(karabo_id, karabo_da, Constants.AGIPD.SlopesPC(),\n",
" condition, cal_db_interface,\n",
@@ -1488,14 +1506,15 @@
" if db_output:\n",
" md = send_to_db(pdu, karabo_id, dbconst, condition,\n",
" file_loc, report, cal_db_interface,\n",
- " creation_time=creation_time)\n",
+ " creation_time=creation_time,\n",
+ " variant=1)\n",
" # TODO: check if this can replace other written function of this notebook.\n",
" #if local_output:\n",
" # md = save_const_to_h5(pdu, karabo_id, dconst, condition, dconst.data, \n",
" # file_loc, report, creation_time, out_folder)\n",
"\n",
"print(\"Constants parameter conditions are:\\n\")\n",
- "print(f\"• memory_cells: {maxcells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
+ "print(f\"• memory_cells: {mem_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
" f\"• acquisition_rate: {acq_rate}\\n• gain_setting: {gain_setting}\\n\"\n",
" f\"• integration_time: {integration_time}\\n\"\n",
" f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
@@ -1507,7 +1526,7 @@
"source": [
"## Overview Plots ##\n",
"\n",
- "Each of the following plots represents one of the fit parameters of memory cell 4 on a module:\n",
+ "Each of the following plots represents one of the fit parameters of a defined memory cell on a module:\n",
"\n",
"For the linear function of the high gain region\n",
"\n",
@@ -1515,9 +1534,17 @@
"\n",
"* ml denotes the $m$ parameter\n",
"* bl denotes the $b$ parameter\n",
- "* devl denotes the anbsolute relative deviation from linearity.\n",
+ "* devl denotes the absolute relative deviation from linearity.\n",
"\n",
- "For the composite function of the medium gain and transition region\n",
+ "For the linear function of the medium gain region\n",
+ "\n",
+ " $$y = mx + b$$\n",
+ "\n",
+ "* mh denotes the $m$ parameter\n",
+ "* bh denotes the $b$ parameter\n",
+ "* devh denotes the absolute relative deviation from linearity.\n",
+ "\n",
+ "For the composite function (if selected) of the medium gain and transition region\n",
" \n",
" $$y = A*e^{-(x-O)/C}+mx+b$$\n",
"\n",
@@ -1525,46 +1552,58 @@
"* ch denotes the $C$ parameter\n",
"* mh denotes the $m$ parameter\n",
"* bh denotes the $b$ parameter\n",
- "* devh denotes the anbsolute relative deviation from the linear part of the function.\n",
+ "* devh denotes the absolute relative deviation from the linear part of the function.\n",
"\n",
"Additionally, the thresholds and bad pixels (mask) are shown.\n",
"\n",
- "Finally, the red and white rectangles indicate the first and second pixel ranges"
+ "Finally, the red and white rectangles indicate the first and second pixel ranges."
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-15T10:22:56.909051Z",
- "start_time": "2019-07-15T10:22:46.983711Z"
- },
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
- "import matplotlib.patches as patches\n",
- "import matplotlib.pyplot as plt\n",
- "from mpl_toolkits.axes_grid1 import AxesGrid\n",
- "\n",
- "cell_to_preview = min(59, mem_cells-1)\n",
- "for module, data in fres.items():\n",
+ "def preview_fitted_params(data, module, cell_to_preview, fit_hook):\n",
+ " plot_text= {\"ml\": 'HG slope (ml)',\n",
+ " \"bl\": 'HG intercept (bl)',\n",
+ " \"devl\": 'HG linearity deviation (devl)',\n",
+ " \"tresh\": 'Gain threshold (tresh)',\n",
+ " \"tresh_bounds\": 'Mean HG signal (tresh_bounds)',\n",
+ " \"dhm\": 'HG boundary (dhm)',\n",
+ " \"mh\": 'MG slope (mh)',\n",
+ " \"bh\": 'MG intercept (bh)',\n",
+ " \"oh\": 'Hook - o (oh)',\n",
+ " \"ch\": 'Hook - c (ch)',\n",
+ " \"ah\": 'Hook - a (ah)',\n",
+ " \"devh\": 'MG linearity deviation (devh)'\n",
+ " }\n",
+ " \n",
+ " mask = bad_pixels[module]\n",
" fig = plt.figure(figsize=(20,20))\n",
- " grid = AxesGrid(fig, 111,\n",
- " nrows_ncols=(7 if fit_hook else 3, 2),\n",
- " axes_pad=(0.9, 0.15),\n",
- " label_mode=\"1\",\n",
+ " if fit_hook:\n",
+ " grid = AxesGrid(fig, 111,\n",
+ " nrows_ncols=(7, 2),\n",
+ " axes_pad=(0.9, 0.55),\n",
+ " label_mode=\"0\",\n",
" share_all=True,\n",
" cbar_location=\"right\",\n",
" cbar_mode=\"each\",\n",
" cbar_size=\"7%\",\n",
- " cbar_pad=\"2%\",\n",
+ " cbar_pad=\"2%\"\n",
+ " )\n",
+ " else:\n",
+ " grid = AxesGrid(fig, 111,\n",
+ " nrows_ncols=(5, 2),\n",
+ " axes_pad=(0.9, 0.55),\n",
+ " label_mode=\"0\",\n",
+ " share_all=True,\n",
+ " cbar_location=\"right\",\n",
+ " cbar_mode=\"each\",\n",
+ " cbar_size=\"7%\",\n",
+ " cbar_pad=\"2%\"\n",
" )\n",
- " \n",
- " \n",
- " mask = bad_pixels[module]\n",
- " \n",
" i = 0\n",
" for key, citem in data.items():\n",
" item = citem.copy()\n",
@@ -1574,49 +1613,74 @@
" maxcnt = 10\n",
" if med < 0:\n",
" bound = -bound\n",
- " \n",
+ "\n",
" while(np.count_nonzero((item < med-bound*med) | (item > med+bound*med))/item.size > 0.01): \n",
- " bound *=2\n",
+ " bound *= 2\n",
" maxcnt -= 1\n",
" if maxcnt < 0:\n",
" break\n",
- " \n",
- " \n",
+ "\n",
+ " \n",
" if \"bounds\" in key:\n",
- " d = item[cell_to_preview,...,0]\n",
- " im = grid[i].imshow(d, interpolation=\"nearest\",\n",
+ " d = item[cell_to_preview, ..., 0]\n",
+ " im = grid[i].imshow(d, interpolation=\"nearest\", origin='lower',\n",
" vmin=med-bound*med, vmax=med+bound*med)\n",
+ " grid[i].set_title(plot_text[key], fontsize=14)\n",
" else:\n",
- " d = item[cell_to_preview,...]\n",
- " im = grid[i].imshow(d, interpolation=\"nearest\",\n",
- " vmin=med-bound*med, vmax=med+bound*med)\n",
+ " if fit_hook:\n",
+ " d = item[cell_to_preview, ...]\n",
+ " im = grid[i].imshow(d, interpolation=\"nearest\", origin='lower',\n",
+ " vmin=med-bound*med, vmax=med+bound*med)\n",
+ " grid[i].set_title(plot_text[key], fontsize=14)\n",
+ " else:\n",
+ " if key == 'oh' or key == 'ch' or key == 'ah':\n",
+ " continue\n",
+ " else:\n",
+ " d = item[cell_to_preview, ...]\n",
+ " im = grid[i].imshow(d, interpolation=\"nearest\", origin='lower',\n",
+ " vmin=med-bound*med, vmax=med+bound*med)\n",
+ " grid[i].set_title(plot_text[key], fontsize=14)\n",
+ " \n",
" cb = grid.cbar_axes[i].colorbar(im)\n",
- " \n",
" # axes coordinates are 0,0 is bottom left and 1,1 is upper right\n",
" x0, x1 = tpix_range1[0][0], tpix_range1[0][1]\n",
" y0, y1 = tpix_range1[1][0], tpix_range1[1][1]\n",
- " p = patches.Rectangle(\n",
- " (x0, y0), x1-x0, y1-y0, fill=False, color=\"red\")\n",
- "\n",
+ " p = patches.Rectangle((x0, y0), (x1 - x0), (y1 - y0), fill=False, color=\"red\")\n",
" grid[i].add_patch(p)\n",
- " \n",
" x0, x1 = tpix_range2[0][0], tpix_range2[0][1]\n",
" y0, y1 = tpix_range2[1][0], tpix_range2[1][1]\n",
- " p = patches.Rectangle(\n",
- " (x0, y0), x1-x0, y1-y0, fill=False, color=\"white\")\n",
- "\n",
+ " p = patches.Rectangle((x0, y0), (x1 - x0), (y1 - y0), fill=False, color=\"white\")\n",
" grid[i].add_patch(p)\n",
- " \n",
- " grid[i].text(20, 50, key, color=\"w\", fontsize=50)\n",
- " \n",
+ "\n",
" i += 1\n",
"\n",
- " im = grid[-1].imshow(mask[cell_to_preview,...], interpolation=\"nearest\",\n",
- " vmin=0, vmax=1)\n",
+ " im = grid[-1].imshow(mask[cell_to_preview, ...], interpolation=\"nearest\", origin='lower', vmin=0, vmax=1)\n",
" cb = grid.cbar_axes[-1].colorbar(im)\n",
+ " grid[-1].set_title('Mask', fontsize=14)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cell_to_preview=[1,mem_cells-5]\n",
"\n",
- " grid[-1].text(20, 50, \"mask\", color=\"w\", fontsize=50)\n",
- " fig.savefig(\"{}/module_{}_PC.png\".format(out_folder, module))"
+ "for module, data in fres.items():\n",
+ " print(' Memory cell {}:'.format(cell_to_preview[0]))\n",
+ " preview_fitted_params(data, module, cell_to_preview[0], fit_hook)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for module, data in fres.items():\n",
+ " print('Memory cell {}:'.format(cell_to_preview[1]))\n",
+ " preview_fitted_params(data, module, cell_to_preview[1], fit_hook)"
]
},
{
@@ -1629,49 +1693,52 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-15T11:03:13.205022Z",
- "start_time": "2019-07-15T11:03:09.623771Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
- "mltomh = ml/mh\n",
- "fres[qm].update({'mltomh': mltomh})\n",
"toplot = {\"tresh\": \"Gain theshold (ADU)\",\n",
" \"ml\": \"Slope (HG)\",\n",
" \"bl\": \"Offset (HG) (ADU)\",\n",
" \"mh\": \"Slope (MG)\",\n",
" \"bh\": \"Offset (MG) (ADU)\",\n",
" \"mltomh\": \"Ration slope_HG/slope_MG\"}\n",
- "from matplotlib.colors import LogNorm, PowerNorm\n",
+ "\n",
+ "fig, ax = plt.subplots(3, 2, figsize=(18, 15))\n",
+ "fig.subplots_adjust(wspace=0.1, hspace=0.15)\n",
"\n",
"for module, data in fres.items():\n",
- " \n",
+ " mltomh = data['ml'] / data['mh']\n",
+ " fres[module].update({'mltomh': mltomh})\n",
" bins = 100\n",
- " \n",
- " for typ, label in toplot.items():\n",
+ "\n",
+ " for counter, (typ, label) in enumerate(toplot.items()):\n",
" r_hist = np.zeros((mem_cells, bins))\n",
" mask = bad_pixels[module]\n",
" thresh = data[typ]\n",
- " hrange = [0.5*np.nanmedian(thresh), 1.5*np.nanmedian(thresh)]\n",
+ " hrange = [0.5 * np.nanmedian(thresh), 1.5 * np.nanmedian(thresh)]\n",
+ " \n",
" if hrange[1] < hrange[0]:\n",
" hrange = hrange[::-1]\n",
" for c in range(mem_cells):\n",
- " d = thresh[c,...]\n",
+ " d = thresh[c, ...]\n",
" h, e = np.histogram(d.flatten(), bins=bins, range=hrange)\n",
" r_hist[c, :] = h\n",
- " fig = plt.figure(figsize=(5,5))\n",
- " ax = fig.add_subplot(111)\n",
- " im = ax.imshow(r_hist[:,:].T[::-1,:], interpolation=\"nearest\",\n",
- " aspect=\"auto\", norm=LogNorm(vmin=1, vmax=np.max(r_hist)),\n",
- " extent=[0, mem_cells, hrange[0], hrange[1]])\n",
- " ax.set_xlabel(\"Memory cell\")\n",
- " ax.set_ylabel(label)\n",
- " cb = fig.colorbar(im)\n",
- " cb.set_label(\"Counts\")\n",
- " #fig.savefig(\"/gpfs/exfel/data/scratch/haufs/test/agipd_gain_threholds.pdf\", bbox_inches=\"tight\")"
+ " \n",
+ " if counter < 3:\n",
+ " im = ax[counter, 0].imshow(r_hist[:, :].T[::-1, :], interpolation=\"nearest\",\n",
+ " aspect=\"auto\", norm=LogNorm(vmin=1, vmax=np.max(r_hist)),\n",
+ " extent=[0, mem_cells, hrange[0], hrange[1]])\n",
+ " ax[counter,0].set_xlabel(\"Memory cell\", fontsize=12)\n",
+ " ax[counter,0].set_ylabel(label, fontsize=12)\n",
+ " cb = fig.colorbar(im, ax=ax[counter, 0], pad=0.01)\n",
+ " else:\n",
+ " im = ax[counter-3, 1].imshow(r_hist[:, :].T[::-1, :], interpolation=\"nearest\",\n",
+ " aspect=\"auto\", norm=LogNorm(vmin=1, vmax=np.max(r_hist)),\n",
+ " extent=[0, mem_cells, hrange[0], hrange[1]])\n",
+ " ax[counter-3, 1].set_xlabel(\"Memory cell\", fontsize=12)\n",
+ " ax[counter-3, 1].set_ylabel(label, fontsize=12)\n",
+ " cb = fig.colorbar(im, ax=ax[counter-3, 1], pad=0.01)\n",
+ " cb.set_label(\"Counts\", fontsize=12) "
]
},
{
@@ -1686,12 +1753,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-15T11:06:12.213906Z",
- "start_time": "2019-07-15T11:03:13.206878Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"cols = {BadPixels.CI_GAIN_OF_OF_THRESHOLD.value: (BadPixels.CI_GAIN_OF_OF_THRESHOLD.name, '#FF000080'),\n",
@@ -1708,15 +1770,10 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-07-16T06:48:28.958002Z",
- "start_time": "2019-07-16T06:48:24.824045Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
- "one_photon = 55 # ADU\n",
+ "one_photon = 73 # ADU (10 keV photon)\n",
"test_pixels = []\n",
"tpix_range1 = [(0,8), (0,8)]\n",
"for i in range(*tpix_range1[0]):\n",
@@ -1747,14 +1804,14 @@
" offset = offsets[mod]\n",
" threshold = thresholds[mod]\n",
"\n",
- " medth = np.nanmean(threshold[...,0])\n",
+ " medth = np.nanmean(threshold[..., 0])\n",
" for pix in test_pixels:\n",
" for cell in test_cells:\n",
- " color = np.random.rand(3,1)\n",
+ " color = np.random.rand(3, 1)\n",
" \n",
" x = np.arange(dig.shape[0])\n",
- " y = dig[:,cell, pix[0], pix[1]]\n",
- " a = ana[:,cell, pix[0], pix[1]]\n",
+ " y = dig[:, cell, pix[0], pix[1]]\n",
+ " a = ana[:, cell, pix[0], pix[1]]\n",
" \n",
" vidx = (y > 1000) & np.isfinite(y)\n",
" x = x[vidx]\n",
@@ -1767,27 +1824,25 @@
" markers = ['o','.','x','v']\n",
" colors = ['b', 'r', 'g', 'k']\n",
" ymin = y.min()\n",
- " \n",
" amin = a[labels[2]].min()\n",
+ " \n",
" for i, lbl in enumerate(labels):\n",
" \n",
" if np.any(lbl):\n",
" if i == 0:\n",
" cm = (cdata['ml'][cell, pix[0], pix[1]]/medml)\n",
- " \n",
" o = offset[pix[0], pix[1], cell, 0]\n",
" ym = (y[lbl]-o)/cm\n",
" \n",
" elif i >= 1:\n",
" mh = cdata['mh'][cell, pix[0], pix[1]]\n",
" ml = cdata['ml'][cell, pix[0], pix[1]]\n",
- " cml = ml/medml\n",
- " cmh = mh/medmh\n",
- " cm = medml/medmh\n",
+ " cml = ml / medml\n",
+ " cmh = mh / medmh\n",
+ " cm = medml / medmh\n",
" oh = cdata['bh'][cell, pix[0], pix[1]]\n",
" o = offset[pix[0], pix[1], cell, 1] + oh\n",
- " \n",
- " ym = (y[lbl]-o)/cmh*cm\n",
+ " ym = (y[lbl]-o) / cmh * cm\n",
" \n",
" if i == 1:\n",
" ah = cdata['ah'][cell, pix[0], pix[1]]\n",
@@ -1808,9 +1863,8 @@
" if np.any(lbl):\n",
" if i == 0:\n",
" cm = (cdata['ml'][cell, pix[0], pix[1]]/medml)\n",
- " \n",
" o = offset[pix[0], pix[1], cell, 0]\n",
- " ym = (y[lbl]-o)/cm\n",
+ " ym = (y[lbl]-o) / cm\n",
" \n",
" elif i >= 1:\n",
" mh = cdata['mh'][cell, pix[0], pix[1]]\n",
@@ -1820,38 +1874,27 @@
" cm = medml/medmh\n",
" oh = cdata['bh'][cell, pix[0], pix[1]]\n",
" o = offset[pix[0], pix[1], cell, 1] + oh\n",
- " \n",
- " ym = (y[lbl]-o)/cmh*cm\n",
+ " ym = (y[lbl]-o) / cmh * cm\n",
" \n",
" if i == 1:\n",
" ah = cdata['ah'][cell, pix[0], pix[1]]\n",
" ch = cdata['ch'][cell, pix[0], pix[1]]\n",
" ohh = cdata['oh'][cell, pix[0], pix[1]]\n",
- " tx = ch * np.log(ah/(y[lbl]-o))+ohh\n",
- " \n",
+ " tx = ch*np.log(ah/(y[lbl]-o)) + ohh\n",
" chook = (ah*np.exp(-(tx-ohh)/ch) - mh*tx)/cmh*cm\n",
" idx = (a[lbl]-amin) < 0\n",
" ym[idx] -= chook[idx]\n",
- " \n",
- " #ym = a[lbl]-amin\n",
" \n",
" h, ex, ey = np.histogram2d(x[lbl], ym/one_photon, range=((0, 600), (0, 15000/one_photon)), bins=(300, 600))\n",
" H2[i] += h\n",
" \n",
- " labels = [a < threshold[pix[0], pix[1], cell,0], a >= threshold[pix[0], pix[1], cell,0]]\n",
+ " labels = [a < threshold[pix[0], pix[1], cell, 0], a >= threshold[pix[0], pix[1], cell, 0]]\n",
" for i, lbl in enumerate(labels):\n",
" \n",
" if np.any(lbl):\n",
- " \n",
- " #if i == 0:\n",
- " # amin = a[lbl].min()\n",
- " #else:\n",
- " # amin = a[labels[0]].min() #a[labels[1]].min()# /(threshold[pix[0], pix[1], cell,0]/medth)\n",
" am = a[lbl] - amin\n",
" h, ex, ea = np.histogram2d(x[lbl], am, range=((0, 600), (-100, 5000)), bins=(300, 400))\n",
- " Ha[i] += h\n",
- " \n",
- " \n",
+ " Ha[i] += h \n",
" \n",
" fig = plt.figure(figsize=(10,15))\n",
" ax = fig.add_subplot(311)\n",
@@ -1863,16 +1906,16 @@
" aspect='auto', cmap='spring', alpha=0.7, vmin=0, vmax=100)\n",
" ax.imshow(H[2].T, origin=\"lower\", extent=[ex[0], ex[-1], ey[0], ey[-1]],\n",
" aspect='auto', cmap='winter', alpha=0.7, vmin=0, vmax=1000)\n",
- " ax.set_ylabel(\"AGIPD response (ADU)\")\n",
+ " ax.set_ylabel(\"AGIPD response (# photon)\")\n",
" ax.set_xlabel(\"PC scan point (#)\")\n",
" \n",
" x = np.arange(0, 600)\n",
- " ideal = medml*x/one_photon\n",
+ " ideal = medml * x / one_photon\n",
" ax.plot(x, ideal, color='red')\n",
" ax.plot(x, ideal + np.sqrt(ideal), color='red')\n",
" ax.plot(x, ideal - np.sqrt(ideal), color='red')\n",
- " \n",
- " \n",
+ " ax.grid(lw=1.5)\n",
+ " \n",
" ax = fig.add_subplot(312)\n",
" for i in range(2):\n",
" H2[i][H2[i]==0] = np.nan \n",
@@ -1880,29 +1923,34 @@
" aspect='auto', cmap='summer', alpha=0.7, vmin=0, vmax=1000)\n",
" ax.imshow(H2[1].T, origin=\"lower\", extent=[ex[0], ex[-1], ey[0], ey[-1]],\n",
" aspect='auto', cmap='winter', alpha=0.7, vmin=0, vmax=1000)\n",
- " ax.set_ylabel(\"AGIPD response (ADU)\")\n",
+ " ax.set_ylabel(\"AGIPD response (# photon)\")\n",
" ax.set_xlabel(\"PC scan point (#)\")\n",
" \n",
" x = np.arange(0, 600)\n",
- " ideal = medml*x/one_photon\n",
+ " ideal = medml * x / one_photon\n",
" ax.plot(x, ideal, color='red')\n",
" ax.plot(x, ideal + np.sqrt(ideal), color='red')\n",
" ax.plot(x, ideal - np.sqrt(ideal), color='red')\n",
- " \n",
- " \n",
+ " ax.grid(lw=1.5)\n",
+ " \n",
" ax = fig.add_subplot(313)\n",
" for i in range(2):\n",
" Ha[i][Ha[i]==0] = np.nan \n",
" ax.imshow(Ha[0].T, origin=\"lower\", extent=[ex[0], ex[-1], ea[0], ea[-1]],\n",
" aspect='auto', cmap='summer', alpha=0.7, vmin=0, vmax=1000)\n",
- " #ax.imshow(Ha[1].T, origin=\"lower\", extent=[ex[0], ex[-1], ea[0], ea[-1]],\n",
- " # aspect='auto', cmap='spring', alpha=0.7, vmin=0, vmax=100)\n",
" ax.imshow(Ha[1].T, origin=\"lower\", extent=[ex[0], ex[-1], ea[0], ea[-1]],\n",
" aspect='auto', cmap='winter', alpha=0.7, vmin=0, vmax=1000)\n",
" ax.set_ylabel(\"AGIPD gain (ADU)\")\n",
" ax.set_xlabel(\"PC scan point (#)\")\n",
- " "
+ " ax.grid(lw=1.5)"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
@@ -1921,7 +1969,24 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.7"
+ "version": "3.8.11"
+ },
+ "latex_envs": {
+ "LaTeX_envs_menu_present": true,
+ "autocomplete": true,
+ "bibliofile": "biblio.bib",
+ "cite_by": "apalike",
+ "current_citInitial": 1,
+ "eqLabelWithNumbers": true,
+ "eqNumInitial": 1,
+ "hotkeys": {
+ "equation": "Ctrl-E",
+ "itemize": "Ctrl-I"
+ },
+ "labels_anchors": false,
+ "latex_user_defs": false,
+ "report_style_numbering": false,
+ "user_envs_cfg": false
}
},
"nbformat": 4,
diff --git a/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..0007b09ba8af3b9577fc359daae5cb908cab85f3
--- /dev/null
+++ b/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
@@ -0,0 +1,562 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "49b6577f-96a5-4dd2-bdd9-da661b2c4619",
+ "metadata": {},
+ "source": [
+ "# Gotthard2 Dark Image Characterization #\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "The following is a processing for offset, noise, and Badpixels maps using dark images taken with Gotthard2 detector.\n",
+ "All constants are evaluated per strip, per pulse, and per memory cell. The maps are calculated for each gain stage that is acquired in 3 separate runs.\n",
+ "\n",
+ "The three maps (calibration constants) can be injected to the database and stored locally."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "818e24e8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/FXE/202231/p900298/raw\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/gotthard2/darks\" # the folder to output to, required\n",
+ "run_high = 7 # run number for G0 dark run, required\n",
+ "run_med = 8 # run number for G1 dark run, required\n",
+ "run_low = 9 # run number for G2 dark run, required\n",
+ "\n",
+ "# Parameters used to access raw data.\n",
+ "karabo_id = \"FXE_XAD_G2XES\" # karabo prefix of Gotthard-II devices\n",
+ "karabo_da = [\"GH201\"] # data aggregators\n",
+ "receiver_template = \"RECEIVER\" # receiver template used to read INSTRUMENT keys.\n",
+ "control_template = \"CONTROL\" # control template used to read CONTROL keys.\n",
+ "instrument_source_template = '{}/DET/{}:daqOutput' # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput' # noqa\n",
+ "ctrl_source_template = '{}/DET/{}' # template for control source name (filled with karabo_id_control)\n",
+ "karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n",
+ "\n",
+ "# Parameters for the calibration database.\n",
+ "use_dir_creation_date = True\n",
+ "cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
+ "cal_db_timeout = 300000 # timeout on caldb requests\n",
+ "overwrite_creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n",
+ "db_output = False # Output constants to the calibration database\n",
+ "local_output = True # Output constants locally\n",
+ "\n",
+ "# Conditions used for injected calibration constants.\n",
+ "bias_voltage = -1 # Detector bias voltage, set to -1 to use value in raw file.\n",
+ "exposure_time = -1. # Detector exposure time, set to -1 to use value in raw file.\n",
+ "exposure_period = -1. # Detector exposure period, set to -1 to use value in raw file.\n",
+ "acquisition_rate = -1. # Detector acquisition rate (1.1/4.5), set to -1 to use value in raw file.\n",
+ "single_photon = -1 # Detector single photon mode (High/Low CDS), set to -1 to use value in raw file.\n",
+ "\n",
+ "# Parameters used during selecting raw data trains.\n",
+ "min_trains = 1 # Minimum number of trains that should be available to process dark constants. Default 1.\n",
+ "max_trains = 1000 # Maximum number of trains to use for processing dark constants. Set to 0 to use all available trains.\n",
+ "badpixel_threshold_sigma = 5. # bad pixels defined by values outside n times this std from median\n",
+ "\n",
+ "# Don't delete! myMDC sends this by default.\n",
+ "operation_mode = '' # Detector dark run acquiring operation mode, optional"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8085f9aa",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "import pasha as psh\n",
+ "from IPython.display import Markdown, display\n",
+ "from extra_data import RunDirectory\n",
+ "from pathlib import Path\n",
+ "\n",
+ "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.gotthard2 import gotthard2algs, gotthard2lib\n",
+ "from cal_tools.step_timing import StepTimer\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_constant_from_db_and_time,\n",
+ " get_pdu_from_db,\n",
+ " get_report,\n",
+ " save_const_to_h5,\n",
+ " send_to_db,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants\n",
+ "\n",
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "18fe4379",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_nums = [run_high, run_med, run_low]\n",
+ "in_folder = Path(in_folder)\n",
+ "out_folder = Path(out_folder)\n",
+ "out_folder.mkdir(exist_ok=True)\n",
+ "\n",
+ "print(f\"Process modules: {karabo_da}\")\n",
+ "\n",
+ "run_dc = RunDirectory(in_folder / f\"r{run_high:04d}\")\n",
+ "file_loc = f\"proposal:{run_dc.run_metadata()['proposalNumber']} runs:{run_high} {run_med} {run_low}\" # noqa\n",
+ "\n",
+ "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
+ "ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)\n",
+ "\n",
+ "# Read report path to associate it later with injected constants.\n",
+ "report = get_report(out_folder)\n",
+ "\n",
+ "\n",
+ "if overwrite_creation_time:\n",
+ " creation_time = datetime.datetime.strptime(\n",
+ " overwrite_creation_time, \"%Y-%m-%d %H:%M:%S.%f\"\n",
+ " )\n",
+ "elif use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run_high)\n",
+ " print(f\"Using {creation_time.isoformat()} as creation time\")\n",
+ "\n",
+ "\n",
+ "if not karabo_id_control:\n",
+ " karabo_id_control = karabo_id"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "108be688",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer = StepTimer()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ff9149fc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Read parameter conditions and validate the values for the three runs.\n",
+ "\n",
+ "step_timer.start()\n",
+ "run_dcs_dict = dict()\n",
+ "\n",
+ "ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)\n",
+ "conditions = {\n",
+ " \"bias_voltage\": set(),\n",
+ " \"exposure_time\": set(),\n",
+ " \"exposure_period\": set(),\n",
+ " \"acquisition_rate\": set(),\n",
+ " \"single_photon\": set(),\n",
+ "}\n",
+ "for gain, run in enumerate(run_nums):\n",
+ " run_dc = RunDirectory(in_folder / f\"r{run:04d}/\")\n",
+ " run_dcs_dict[run] = [gain, run_dc]\n",
+ "\n",
+ " # Read slow data.\n",
+ " g2ctrl = gotthard2lib.Gotthard2Ctrl(run_dc=run_dc, ctrl_src=ctrl_src)\n",
+ " conditions[\"bias_voltage\"].add(\n",
+ " g2ctrl.get_bias_voltage() if bias_voltage == -1 else bias_voltage\n",
+ " )\n",
+ " conditions[\"exposure_time\"].add(\n",
+ " g2ctrl.get_exposure_time() if exposure_time == -1 else exposure_time\n",
+ " )\n",
+ " conditions[\"exposure_period\"].add(\n",
+ " g2ctrl.get_exposure_period() if exposure_period == -1 else exposure_period\n",
+ " )\n",
+ " conditions[\"single_photon\"].add(\n",
+ " g2ctrl.get_single_photon() if single_photon == -1 else single_photon\n",
+ " )\n",
+ " conditions[\"acquisition_rate\"].add(\n",
+ " g2ctrl.get_acquisition_rate() if acquisition_rate == -1 else acquisition_rate\n",
+ " )\n",
+ "\n",
+ "for c, v in conditions.items():\n",
+ " assert len(v) == 1, f\"{c} value is not the same for the three runs.\"\n",
+ "\n",
+ "bias_voltage = conditions[\"bias_voltage\"].pop()\n",
+ "print(\"Bias voltage: \", bias_voltage)\n",
+ "exposure_time = conditions[\"exposure_time\"].pop()\n",
+ "print(\"Exposure time: \", exposure_time)\n",
+ "exposure_period = conditions[\"exposure_period\"].pop()\n",
+ "print(\"Exposure period: \", exposure_period)\n",
+ "single_photon = conditions[\"single_photon\"].pop()\n",
+ "print(\"Single photon: \", single_photon)\n",
+ "acquisition_rate = conditions[\"acquisition_rate\"].pop()\n",
+ "print(\"Acquisition rate: \", acquisition_rate)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ac9c5dc3-bc66-4e7e-b6a1-360259be535c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def specify_trains_to_process(\n",
+ " img_key_data: \"extra_data.KeyData\", # noqa\n",
+ " max_trains: int = 0,\n",
+ " min_trains: int = 0,\n",
+ "):\n",
+ " \"\"\"Specify total number of trains to process.\n",
+ " Based on given min_trains and max_trains, if given.\n",
+ "\n",
+ " Print number of trains to process and number of empty trains.\n",
+ " Raise ValueError if specified trains are less than min_trains.\n",
+ " \"\"\"\n",
+ " # Specifies total number of trains to proccess.\n",
+ " n_trains = img_key_data.shape[0]\n",
+ " all_trains = len(img_key_data.train_ids)\n",
+ " print(\n",
+ " f\"{mod} has {all_trains - n_trains} \"\n",
+ " f\"trains with empty frames out of {all_trains} trains\"\n",
+ " )\n",
+ "\n",
+ " if n_trains < min_trains:\n",
+ " raise ValueError(\n",
+ " f\"Less than {min_trains} trains are available in RAW data.\"\n",
+ " \" Not enough data to process darks.\"\n",
+ " )\n",
+ "\n",
+ " if max_trains > 0:\n",
+ " n_trains = min(n_trains, max_trains)\n",
+ "\n",
+ " print(f\"Processing {n_trains} trains.\")\n",
+ "\n",
+ " return n_trains"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3c59c11d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# set the operating condition\n",
+ "condition = Conditions.Dark.Gotthard2(\n",
+ " bias_voltage=bias_voltage,\n",
+ " exposure_time=exposure_time,\n",
+ " exposure_period=exposure_period,\n",
+ " acquisition_rate=acquisition_rate,\n",
+ " single_photon=single_photon,\n",
+ ")\n",
+ "\n",
+ "db_modules = get_pdu_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " constant=Constants.Gotthard2.LUT(),\n",
+ " condition=condition,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " snapshot_at=creation_time,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e2eb2fc0-df9c-4887-9691-f81474f8c131",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def convert_train(wid, index, tid, d):\n",
+ " \"\"\"Convert a Gotthard2 train from 12bit to 10bit.\"\"\"\n",
+ " gotthard2algs.convert_to_10bit(\n",
+ " d[instr_mod_src][\"data.adc\"], lut, data_10bit[index, ...]\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4e8ffeae",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Calculate noise and offset per pixel and global average, std and median\n",
+ "noise_map = dict()\n",
+ "offset_map = dict()\n",
+ "badpixels_map = dict()\n",
+ "context = psh.ProcessContext(num_workers=25)\n",
+ "\n",
+ "empty_lut = (np.arange(2 ** 12).astype(np.float64) * 2 ** 10 / 2 ** 12).astype(\n",
+ " np.uint16\n",
+ ")\n",
+ "empty_lut = np.stack(1280 * [np.stack([empty_lut] * 2)], axis=0)\n",
+ "for mod in karabo_da:\n",
+ "\n",
+ " # Retrieve LUT constant\n",
+ " lut, time = get_constant_from_db_and_time(\n",
+ " constant=Constants.Gotthard2.LUT(),\n",
+ " condition=condition,\n",
+ " empty_constant=empty_lut,\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=mod,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout,\n",
+ " print_once=False,\n",
+ " )\n",
+ " print(f\"Retrieved LUT constant with creation-time {time}\")\n",
+ " # Path to pixels ADC values\n",
+ " instr_mod_src = instrument_src.format(int(mod[-2:]))\n",
+ "\n",
+ " # TODO: Validate the final shape to store constants.\n",
+ " cshape = (1280, 2, 3)\n",
+ "\n",
+ " offset_map[mod] = context.alloc(shape=cshape, dtype=np.float32)\n",
+ " noise_map[mod] = context.alloc(like=offset_map[mod])\n",
+ " badpixels_map[mod] = context.alloc(like=offset_map[mod], dtype=np.uint32)\n",
+ "\n",
+ " for run_num, [gain, run_dc] in run_dcs_dict.items():\n",
+ " step_timer.start()\n",
+ " n_trains = specify_trains_to_process(run_dc[instr_mod_src, \"data.adc\"])\n",
+ "\n",
+ " # Select requested number of trains to process.\n",
+ " dc = run_dc.select(instr_mod_src, require_all=True).select_trains(\n",
+ " np.s_[:n_trains]\n",
+ " )\n",
+ "\n",
+ " step_timer.done_step(\"preparing raw data\")\n",
+ "\n",
+ " step_timer.start()\n",
+ " # Convert 12bit data to 10bit\n",
+ " data_10bit = context.alloc(\n",
+ " shape=dc[instr_mod_src, \"data.adc\"].shape, dtype=np.float32\n",
+ " )\n",
+ " context.map(convert_train, dc)\n",
+ " step_timer.done_step(\"convert to 10bit\")\n",
+ "\n",
+ " step_timer.start()\n",
+ "\n",
+ " # The first ~20 frames of each train are excluded.\n",
+ " # The electronics needs some time to reach stable operational conditions\n",
+ " # at the beginning of each acquisition cycle,\n",
+ " # hence the first ~20 images have lower quality and should not be used.\n",
+ " # The rough number of 20 is correct at 4.5 MHz acquisition rate,\n",
+ " # 5 should be enough at 1.1 MHz. Sticking with 20 excluded frames, as there isn't\n",
+ " # much of expected difference.\n",
+ "\n",
+ " # Split even and odd data to calculate the two storage cell constants.\n",
+ " # Detector always operates in burst mode.\n",
+ " even_data = data_10bit[:, 20::2, :]\n",
+ " odd_data = data_10bit[:, 21::2, :]\n",
+ "\n",
+ " def offset_noise_cell(wid, index, d):\n",
+ " offset_map[mod][:, index, gain] = np.mean(d, axis=(0, 1))\n",
+ " noise_map[mod][:, index, gain] = np.std(d, axis=(0, 1))\n",
+ "\n",
+ " # Calculate Offset and Noise Maps.\n",
+ " context.map(offset_noise_cell, (even_data, odd_data))\n",
+ "\n",
+ " # Split even and odd gain data.\n",
+ " data_gain = dc[instr_mod_src, \"data.gain\"].ndarray()\n",
+ " even_gain = data_gain[:, 20::2, :]\n",
+ " odd_gain = data_gain[:, 21::2, :]\n",
+ " raw_g = 3 if gain == 2 else gain\n",
+ "\n",
+ " def badpixels_cell(wid, index, g):\n",
+ " \"\"\"Check if there are wrong bad gain values.\n",
+ " Indicate pixels with wrong gain value across all trains for each cell.\"\"\"\n",
+ " badpixels_map[mod][\n",
+ " np.mean(g, axis=(0, 1)) != raw_g, index, gain\n",
+ " ] |= BadPixels.WRONG_GAIN_VALUE.value\n",
+ "\n",
+ " context.map(badpixels_cell, (even_gain, odd_gain))\n",
+ "\n",
+ " step_timer.done_step(\"Processing darks\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8e410ca2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"Total processing time {step_timer.timespan():.01f} s\")\n",
+ "step_timer.print_summary()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3fc17e05-17ab-4ac4-9e79-c95399278bb9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def print_bp_entry(bp):\n",
+ " print(f\"{bp.name:<30s} {bp.value:032b} -> {int(bp.value)}\")\n",
+ "\n",
+ "\n",
+ "print_bp_entry(BadPixels.NOISE_OUT_OF_THRESHOLD)\n",
+ "print_bp_entry(BadPixels.OFFSET_NOISE_EVAL_ERROR)\n",
+ "print_bp_entry(BadPixels.WRONG_GAIN_VALUE)\n",
+ "\n",
+ "\n",
+ "def eval_bpidx(d):\n",
+ " mdn = np.nanmedian(d, axis=(0))[None, :, :]\n",
+ " std = np.nanstd(d, axis=(0))[None, :, :]\n",
+ " idx = (d > badpixel_threshold_sigma * std + mdn) | (\n",
+ " d < (-badpixel_threshold_sigma) * std + mdn\n",
+ " )\n",
+ " return idx"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "40c34cc5-fe93-4b83-bf39-f465f37c40b4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "g_name = [\"G0\", \"G1\", \"G2\"]\n",
+ "\n",
+ "for mod, pdu in zip(karabo_da, db_modules):\n",
+ " display(Markdown(f\"### Badpixels for module {mod}:\"))\n",
+ "\n",
+ " badpixels_map[mod][\n",
+ " ~np.isfinite(offset_map[mod])\n",
+ " ] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
+ " badpixels_map[mod][\n",
+ " eval_bpidx(noise_map[mod])\n",
+ " ] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
+ "\n",
+ " badpixels_map[mod][\n",
+ " ~np.isfinite(noise_map[mod])\n",
+ " ] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
+ "\n",
+ " for cell in [0, 1]:\n",
+ " fig, ax = plt.subplots(figsize=(10, 5))\n",
+ " for g_idx in [0, 1, 2]:\n",
+ " ax.plot(badpixels_map[mod][:, cell, g_idx], label=f\"G{g_idx} Bad pixel map\")\n",
+ " ax.set_xticks(np.arange(0, 1281, 80))\n",
+ " ax.set_xlabel(\"Stripes #\")\n",
+ " ax.set_xlabel(\"BadPixels\")\n",
+ " ax.set_title(f\"Cell {cell} - Module {mod} ({pdu})\")\n",
+ " ax.set_ylim([0, 5])\n",
+ " ax.legend()\n",
+ " pass\n",
+ "step_timer.done_step(f\"Creating bad pixels constant and plotting it.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c8777cfe",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for mod, pdu in zip(karabo_da, db_modules):\n",
+ " for cons, cname in zip([offset_map, noise_map], [\"Offset\", \"Noise\"]):\n",
+ " for cell in [0, 1]:\n",
+ " fig, ax = plt.subplots(figsize=(10, 5))\n",
+ " for g_idx in [0, 1, 2]:\n",
+ " ax.plot(cons[mod][:, cell, g_idx], label=f\"G{g_idx} {cname} map\")\n",
+ "\n",
+ " ax.set_xlabel(\"Stripes #\")\n",
+ " ax.set_xlabel(cname)\n",
+ " ax.set_title(f\"{cname} map - Cell {cell} - Module {mod} ({pdu})\")\n",
+ " ax.legend()\n",
+ " pass"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1c4eddf7-7d6e-49f4-8cbb-12d2bc496a8f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "for mod, db_mod in zip(karabo_da, db_modules):\n",
+ " constants = {\n",
+ " \"Offset\": offset_map[mod],\n",
+ " \"Noise\": noise_map[mod],\n",
+ " \"BadPixelsDark\": badpixels_map[mod],\n",
+ " }\n",
+ "\n",
+ " md = None\n",
+ "\n",
+ " for key, const_data in constants.items():\n",
+ "\n",
+ " const = getattr(Constants.Gotthard2, key)()\n",
+ " const.data = const_data\n",
+ "\n",
+ " if db_output:\n",
+ " md = send_to_db(\n",
+ " db_module=db_mod,\n",
+ " karabo_id=karabo_id,\n",
+ " constant=const,\n",
+ " condition=condition,\n",
+ " file_loc=file_loc,\n",
+ " report_path=report,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout,\n",
+ " )\n",
+ " if local_output:\n",
+ " md = save_const_to_h5(\n",
+ " db_module=db_mod,\n",
+ " karabo_id=karabo_id,\n",
+ " constant=const,\n",
+ " condition=condition,\n",
+ " data=const.data,\n",
+ " file_loc=file_loc,\n",
+ " report=report,\n",
+ " creation_time=creation_time,\n",
+ " out_folder=out_folder,\n",
+ " )\n",
+ " print(f\"Calibration constant {key} is stored locally at {out_folder}.\\n\")\n",
+ "\n",
+ "print(\"Constants parameter conditions are:\\n\")\n",
+ "print(\n",
+ " f\"• Bias voltage: {bias_voltage}\\n\"\n",
+ " f\"• Exposure time: {exposure_time}\\n\"\n",
+ " f\"• Exposure period: {exposure_period}\\n\"\n",
+ " f\"• Acquisition rate: {acquisition_rate}\\n\"\n",
+ " f\"• Single photon: {single_photon}\\n\"\n",
+ " f\"• Creation time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\"\n",
+ ")\n",
+ "step_timer.done_step(\"Injecting constants.\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.8.11 ('.cal4_venv': venv)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "25ceec0b6126c0ccf883616a02d86b8eaec8ca3fe33700925044adbe0a704e39"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..fc8913230eacddea36f4c21b9ab1f557455bf743
--- /dev/null
+++ b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
@@ -0,0 +1,634 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "bed7bd15-21d9-4735-82c1-c27c1a5e3346",
+ "metadata": {},
+ "source": [
+ "# Gotthard2 Offline Correction #\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "Offline Calibration for the Gothard2 Detector"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "570322ed-f611-4fd1-b2ec-c12c13d55843",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/FXE/202221/p003225/raw\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/gotthard2\" # the folder to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
+ "run = 50 # run to process, required\n",
+ "sequences = [-1] # sequences to correct, set to [-1] for all, range allowed\n",
+ "sequences_per_node = 1 # number of sequence files per node if notebook executed through xfel-calibrate, set to 0 to not run SLURM parallel\n",
+ "\n",
+ "# Parameters used to access raw data.\n",
+ "karabo_id = \"FXE_XAD_G2XES\" # karabo prefix of Gotthard-II devices\n",
+ "karabo_da = [\"GH201\"] # data aggregators\n",
+ "receiver_template = \"RECEIVER\" # receiver template used to read INSTRUMENT keys.\n",
+ "control_template = \"CONTROL\" # control template used to read CONTROL keys.\n",
+ "instrument_source_template = \"{}/DET/{}:daqOutput\" # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'\n",
+ "ctrl_source_template = \"{}/DET/{}\" # template for control source name (filled with karabo_id_control)\n",
+ "karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n",
+ "\n",
+ "# Parameters for calibration database.\n",
+ "use_dir_creation_date = True # use the creation data of the input dir for database queries.\n",
+ "cal_db_interface = \"tcp://max-exfl016:8016#8025\" # the database interface to use.\n",
+ "cal_db_timeout = 180000 # timeout on caldb requests.\n",
+ "overwrite_creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n",
+ "\n",
+ "# Parameters affecting corrected data.\n",
+ "constants_file = \"\" # Use constants in given constant file path. /gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\n",
+ "offset_correction = True # apply offset correction. This can be disabled to only apply LUT or apply LUT and gain correction for non-linear differential results.\n",
+ "gain_correction = True # apply gain correction.\n",
+ "\n",
+ "# Parameter conditions.\n",
+ "bias_voltage = -1 # Detector bias voltage, set to -1 to use value in raw file.\n",
+ "exposure_time = -1. # Detector exposure time, set to -1 to use value in raw file.\n",
+ "exposure_period = -1. # Detector exposure period, set to -1 to use value in raw file.\n",
+ "acquisition_rate = -1. # Detector acquisition rate (1.1/4.5), set to -1 to use value in raw file.\n",
+ "single_photon = -1 # Detector single photon mode (High/Low CDS), set to -1 to use value in raw file.\n",
+ "\n",
+ "# Parameters for plotting\n",
+ "skip_plots = False # exit after writing corrected files\n",
+ "pulse_idx_preview = 3 # pulse index to preview. The following even/odd pulse index is used for preview. # TODO: update to pulseId preview.\n",
+ "\n",
+ "\n",
+ "def balance_sequences(in_folder, run, sequences, sequences_per_node, karabo_da):\n",
+ " from xfel_calibrate.calibrate import balance_sequences as bs\n",
+ " return bs(in_folder, run, sequences, sequences_per_node, karabo_da)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6e9730d8-3908-41d7-abe2-d78e046d5de2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "import warnings\n",
+ "from functools import partial\n",
+ "\n",
+ "import h5py\n",
+ "import pasha as psh\n",
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "from IPython.display import Markdown, display\n",
+ "from extra_data import RunDirectory, H5File\n",
+ "from pathlib import Path\n",
+ "\n",
+ "from cal_tools import h5_copy_except\n",
+ "from cal_tools.gotthard2 import gotthard2algs, gotthard2lib\n",
+ "from cal_tools.step_timing import StepTimer\n",
+ "from cal_tools.tools import (\n",
+ " get_constant_from_db_and_time,\n",
+ " get_dir_creation_date,\n",
+ " get_pdu_from_db,\n",
+ " CalibrationMetadata,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants\n",
+ "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
+ "\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d7c02c48-4429-42ea-a42e-de45366d7fa3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = Path(in_folder)\n",
+ "run_folder = in_folder / f\"r{run:04d}\"\n",
+ "out_folder = Path(out_folder)\n",
+ "out_folder.mkdir(parents=True, exist_ok=True)\n",
+ "\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# NOTE: this notebook will not overwrite calibration metadata file\n",
+ "const_yaml = metadata.get(\"retrieved-constants\", {})\n",
+ "\n",
+ "if not karabo_id_control:\n",
+ " karabo_id_control = karabo_id\n",
+ "\n",
+ "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
+ "ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)\n",
+ "\n",
+ "print(f\"Process modules: {karabo_da} for run {run}\")\n",
+ "\n",
+ "creation_time = None\n",
+ "if overwrite_creation_time:\n",
+ " creation_time = datetime.datetime.strptime(\n",
+ " overwrite_creation_time, \"%Y-%m-%d %H:%M:%S.%f\"\n",
+ " )\n",
+ "elif use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run)\n",
+ " print(f\"Using {creation_time} as creation time\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b5eb816e-b5f2-44ce-9907-0273d82341b6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Select only sequence files to process for the selected detector.\n",
+ "if sequences == [-1]:\n",
+ " possible_patterns = list(f\"*{mod}*.h5\" for mod in karabo_da)\n",
+ "else:\n",
+ " possible_patterns = list(\n",
+ " f\"*{mod}-S{s:05d}.h5\" for mod in karabo_da for s in sequences\n",
+ " )\n",
+ "\n",
+ "run_folder = Path(in_folder / f\"r{run:04d}\")\n",
+ "seq_files = [\n",
+ " f for f in run_folder.glob(\"*.h5\") if any(f.match(p) for p in possible_patterns)\n",
+ "]\n",
+ "\n",
+ "seq_files = sorted(seq_files)\n",
+ "\n",
+ "if not seq_files:\n",
+ " raise IndexError(\"No sequence files available for the selected sequences.\")\n",
+ "\n",
+ "print(f\"Processing a total of {len(seq_files)} sequence files\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f9a8d1eb-ce6a-4ed0-abf4-4a6029734672",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer = StepTimer()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "892172d8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Read slow data\n",
+ "run_dc = RunDirectory(run_folder)\n",
+ "g2ctrl = gotthard2lib.Gotthard2Ctrl(run_dc=run_dc, ctrl_src=ctrl_src)\n",
+ "\n",
+ "if bias_voltage == -1:\n",
+ " bias_voltage = g2ctrl.get_bias_voltage()\n",
+ "if exposure_time == -1:\n",
+ " exposure_time = g2ctrl.get_exposure_time()\n",
+ "if exposure_period == -1:\n",
+ " exposure_period = g2ctrl.get_exposure_period()\n",
+ "if acquisition_rate == -1:\n",
+ " acquisition_rate = g2ctrl.get_acquisition_rate()\n",
+ "if single_photon == -1:\n",
+ " single_photon = g2ctrl.get_single_photon()\n",
+ "\n",
+ "print(\"Bias Voltage:\", bias_voltage)\n",
+ "print(\"Exposure Time:\", exposure_time)\n",
+ "print(\"Exposure Period:\", exposure_period)\n",
+ "print(\"Acquisition Rate:\", acquisition_rate)\n",
+ "print(\"Single Photon:\", single_photon)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8c852392-bb19-4c40-b2ce-3b787538a92d",
+ "metadata": {},
+ "source": [
+ "### Retrieving calibration constants"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5717d722",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Used for old FXE (p003225) runs before adding Gotthard2 to CALCAT\n",
+ "const_data = dict()\n",
+ "if constants_file:\n",
+ " for mod in karabo_da:\n",
+ " const_data[mod] = dict()\n",
+ " # load constants temporarily using defined local paths.\n",
+ " with h5py.File(constants_file, \"r\") as cfile:\n",
+ " const_data[mod][\"LUT\"] = cfile[\"LUT\"][()]\n",
+ " const_data[mod][\"Offset\"] = cfile[\"offset_map\"][()].astype(np.float32)\n",
+ " const_data[mod][\"RelativeGain\"] = cfile[\"gain_map\"][()].astype(np.float32)\n",
+ " const_data[mod][\"Mask\"] = cfile[\"bpix_ff\"][()].astype(np.uint32)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1cdbe818",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Conditions iCalibrationDB object.\n",
+ "condition = Conditions.Dark.Gotthard2(\n",
+ " bias_voltage=bias_voltage,\n",
+ " exposure_time=exposure_time,\n",
+ " exposure_period=exposure_period,\n",
+ " single_photon=single_photon,\n",
+ " acquisition_rate=acquisition_rate,\n",
+ ")\n",
+ "\n",
+ "# TODO: Maybe this condition and previous cell can be removed later after the initial phase.\n",
+ "if not constants_file:\n",
+ " # Prepare a dictionary of empty constants to loop on\n",
+ " # it's keys and initiate non-retrieved constants.\n",
+ " empty_lut = (np.arange(2 ** 12).astype(np.float64) * 2 ** 10 / 2 ** 12).astype(\n",
+ " np.uint16\n",
+ " )\n",
+ " empty_lut = np.stack(1280 * [np.stack([empty_lut] * 2)], axis=0)\n",
+ " empty_constants = {\n",
+ " \"LUT\": empty_lut,\n",
+ " \"Offset\": np.zeros((1280, 2, 3), dtype=np.float32),\n",
+ " \"BadPixelsDark\": np.zeros((1280, 2, 3), dtype=np.uint32),\n",
+ " \"RelativeGain\": np.ones((1280, 2, 3), dtype=np.float32),\n",
+ " \"BadPixelsFF\": np.zeros((1280, 2, 3), dtype=np.uint32),\n",
+ " }\n",
+ "\n",
+ " for mod in karabo_da:\n",
+ " const_data[mod] = dict()\n",
+ " # Only used for printing timestamps within the loop.\n",
+ " when = dict()\n",
+ " # Check YAML file for constant metadata of file path and creation-time\n",
+ " if const_yaml:\n",
+ " for cname, mdata in const_yaml[mod][\"constants\"].items():\n",
+ " const_data[mod][cname] = dict()\n",
+ " when[cname] = mdata[\"creation-time\"]\n",
+ " if when[cname]:\n",
+ " with h5py.File(mdata[\"file-path\"], \"r\") as cf:\n",
+ " const_data[mod][cname] = np.copy(\n",
+ " cf[f\"{mdata['dataset-name']}/data\"]\n",
+ " )\n",
+ " else:\n",
+ " const_data[mod][cname] = empty_constants[cname]\n",
+ " else: # Retrieve constants from CALCAT. Missing YAML file or running notebook interactively.\n",
+ " for cname, cempty in empty_constants.items():\n",
+ " const_data[mod][cname] = dict()\n",
+ " const_data[mod][cname], when[cname] = get_constant_from_db_and_time(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=mod,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout,\n",
+ " print_once=False,\n",
+ " condition=condition,\n",
+ " constant=getattr(Constants.Gotthard2, cname)(),\n",
+ " empty_constant=cempty,\n",
+ " )\n",
+ " bpix = const_data[mod][\"BadPixelsDark\"]\n",
+ " bpix |= const_data[mod][\"BadPixelsFF\"]\n",
+ " const_data[mod][\"Mask\"] = bpix\n",
+ "\n",
+ " # Print timestamps for the retrieved constants.\n",
+ " print(f\"Constants for module {mod}:\")\n",
+ " for cname, ctime in when.items():\n",
+ " print(f\" {cname} injected at {ctime}\")\n",
+ " del when"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "23fcf7f4-351a-4df7-8829-d8497d94fecc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "context = psh.ProcessContext(num_workers=23)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "daecd662-26d2-4cb8-aa70-383a579cf9f9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def correct_train(wid, index, d):\n",
+ " g = gain[index]\n",
+ " gotthard2algs.convert_to_10bit(d, const_data[mod][\"LUT\"], data_corr[index, ...])\n",
+ " gotthard2algs.correct_train(\n",
+ " data_corr[index, ...],\n",
+ " mask[index, ...],\n",
+ " g,\n",
+ " const_data[mod][\"Offset\"],\n",
+ " const_data[mod][\"RelativeGain\"],\n",
+ " const_data[mod][\"Mask\"],\n",
+ " apply_offset=offset_correction,\n",
+ " apply_gain=gain_correction,\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f88c1aa6-a735-4b72-adce-b30162f5daea",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for mod in karabo_da:\n",
+ " # This is used in case receiver template consists of\n",
+ " # karabo data aggregator index. e.g. detector at DETLAB\n",
+ " instr_mod_src = instrument_src.format(mod[-2:])\n",
+ " data_path = \"INSTRUMENT/\" + instr_mod_src + \"/data\"\n",
+ " for raw_file in seq_files:\n",
+ " step_timer.start()\n",
+ "\n",
+ " dc = H5File(raw_file)\n",
+ " out_file = out_folder / raw_file.name.replace(\"RAW\", \"CORR\")\n",
+ "\n",
+ " # Select module INSTRUMENT source and deselect empty trains.\n",
+ " dc = dc.select(instr_mod_src, require_all=True)\n",
+ " data = dc[instr_mod_src, \"data.adc\"].ndarray()\n",
+ " gain = dc[instr_mod_src, \"data.gain\"].ndarray()\n",
+ " step_timer.done_step(\"preparing raw data\")\n",
+ " dshape = data.shape\n",
+ "\n",
+ " step_timer.start()\n",
+ "\n",
+ " # Allocate shared arrays.\n",
+ " data_corr = context.alloc(shape=dshape, dtype=np.float32)\n",
+ " mask = context.alloc(shape=dshape, dtype=np.uint32)\n",
+ " context.map(correct_train, data)\n",
+ " step_timer.done_step(\"Correcting one sequence file\")\n",
+ "\n",
+ " step_timer.start()\n",
+ "\n",
+ " # Provided PSI gain map has 0 values. Set inf values to nan.\n",
+ " # TODO: This can maybe be removed after creating XFEL gain maps.?\n",
+ " data_corr[np.isinf(data_corr)] = np.nan\n",
+ " # Create CORR files and add corrected data sources.\n",
+ " # Exclude raw data images (data/adc)\n",
+ " with h5py.File(out_file, \"w\") as ofile:\n",
+ " # Copy RAW non-calibrated sources.\n",
+ " with h5py.File(raw_file, \"r\") as sfile:\n",
+ " h5_copy_except.h5_copy_except_paths(sfile, ofile, [f\"{data_path}/adc\"])\n",
+ " # Create datasets with the available corrected data\n",
+ " ddset = ofile.create_dataset(\n",
+ " f\"{data_path}/adc\",\n",
+ " data=data_corr,\n",
+ " chunks=((1,) + dshape[1:]), # 1 chunk == 1 image\n",
+ " dtype=np.float32,\n",
+ " )\n",
+ " # Create datasets with the available corrected data\n",
+ " ddset = ofile.create_dataset(\n",
+ " f\"{data_path}/mask\",\n",
+ " data=mask,\n",
+ " chunks=((1,) + dshape[1:]), # 1 chunk == 1 image\n",
+ " dtype=np.uint32,\n",
+ " compression=\"gzip\",\n",
+ " compression_opts=1,\n",
+ " shuffle=True,\n",
+ " )\n",
+ " step_timer.done_step(\"Storing data\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "94b8e4d2-9f8c-4c23-a509-39238dd8435c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"Total processing time {step_timer.timespan():.01f} s\")\n",
+ "step_timer.print_summary()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0ccc7f7e-2a3f-4ac0-b854-7d505410d2fd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if skip_plots:\n",
+ " print(\"Skipping plots\")\n",
+ " import sys\n",
+ "\n",
+ " sys.exit(0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ff203f77-3811-46f3-bf7d-226d2dcab13f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mod_dcs = {}\n",
+ "first_seq_raw = seq_files[0]\n",
+ "first_seq_corr = out_folder / first_seq_raw.name.replace(\"RAW\", \"CORR\")\n",
+ "for mod in karabo_da:\n",
+ " mod_dcs[mod] = {}\n",
+ " with H5File(first_seq_corr) as out_dc:\n",
+ " tid, mod_dcs[mod][\"train_corr_data\"] = next(\n",
+ " out_dc[instr_mod_src, \"data.adc\"].trains()\n",
+ " )\n",
+ " with H5File(first_seq_raw) as in_dc:\n",
+ " train_dict = in_dc.train_from_id(tid)[1][instr_mod_src]\n",
+ " mod_dcs[mod][\"train_raw_data\"] = train_dict[\"data.adc\"]\n",
+ " mod_dcs[mod][\"train_raw_gain\"] = train_dict[\"data.gain\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "494b453a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Keep as long as it is essential to correct\n",
+ "# RAW data (FXE p003225) before the data mapping was added to CALCAT.\n",
+ "try:\n",
+ " db_modules = get_pdu_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " constant=Constants.jungfrau.Offset(),\n",
+ " condition=condition,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " snapshot_at=creation_time,\n",
+ " )\n",
+ "except RuntimeError:\n",
+ " print(\n",
+ " \"No Physical detector units found for this\"\n",
+ " \" detector mapping at the RAW data creation time.\"\n",
+ " )\n",
+ " db_modules = [None] * len(karabo_da)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1b379438-eb1d-42b2-ac83-eb8cf88c46db",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown(\"### Mean RAW and CORRECTED across pulses for one train:\"))\n",
+ "display(Markdown(f\"Train: {tid}\"))\n",
+ "\n",
+ "step_timer.start()\n",
+ "for mod, pdu in zip(karabo_da, db_modules):\n",
+ "\n",
+ " fig, ax = plt.subplots(figsize=(20, 10))\n",
+ " raw_data = mod_dcs[mod][\"train_raw_data\"]\n",
+ " im = ax.plot(np.mean(raw_data, axis=0))\n",
+ " ax.set_title(f\"RAW module {mod} ({pdu})\")\n",
+ " ax.set_xlabel(\"Strip #\", size=20)\n",
+ " ax.set_ylabel(\"12-bit ADC output\", size=20)\n",
+ " plt.xticks(fontsize=20)\n",
+ " plt.yticks(fontsize=20)\n",
+ " pass\n",
+ "\n",
+ " fig, ax = plt.subplots(figsize=(20, 10))\n",
+ " corr_data = mod_dcs[mod][\"train_corr_data\"]\n",
+ " im = ax.plot(np.mean(corr_data, axis=0))\n",
+ " ax.set_title(f\"CORRECTED module {mod} ({pdu})\")\n",
+ " ax.set_xlabel(\"Strip #\", size=20)\n",
+ " ax.set_ylabel(\"10-bit KeV. output\", size=20)\n",
+ " plt.xticks(fontsize=20)\n",
+ " plt.yticks(fontsize=20)\n",
+ " pass\n",
+ "step_timer.done_step(\"Plotting mean data\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "58a6a276",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown(f\"### RAW and CORRECTED strips across pulses for train {tid}\"))\n",
+ "\n",
+ "step_timer.start()\n",
+ "for mod, pdu in zip(karabo_da, db_modules):\n",
+ " for plt_data, dname in zip(\n",
+ " [\"train_raw_data\", \"train_corr_data\"], [\"RAW\", \"CORRECTED\"]\n",
+ " ):\n",
+ " fig, ax = plt.subplots(figsize=(15, 20))\n",
+ " plt.rcParams.update({\"font.size\": 20})\n",
+ "\n",
+ " heatmapPlot(\n",
+ " mod_dcs[mod][plt_data],\n",
+ " y_label=\"Pulses\",\n",
+ " x_label=\"Strips\",\n",
+ " title=f\"{dname} module {mod} ({pdu})\",\n",
+ " use_axis=ax,\n",
+ " )\n",
+ " pass\n",
+ "step_timer.done_step(\"Plotting RAW and CORRECTED data for one train\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cd8f5e08-fcee-4bff-ba63-6452b3d892a2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Validate given \"pulse_idx_preview\"\n",
+ "\n",
+ "if pulse_idx_preview + 1 > data.shape[1]:\n",
+ " print(\n",
+ " f\"WARNING: selected pulse_idx_preview {pulse_idx_preview} is not available in data.\"\n",
+ " \" Previewing 1st pulse.\"\n",
+ " )\n",
+ " pulse_idx_preview = 1\n",
+ "\n",
+ "if data.shape[1] == 1:\n",
+ " odd_pulse = 1\n",
+ " even_pulse = None\n",
+ "else:\n",
+ " odd_pulse = pulse_idx_preview if pulse_idx_preview % 2 else pulse_idx_preview + 1\n",
+ " even_pulse = (\n",
+ " pulse_idx_preview if not (pulse_idx_preview % 2) else pulse_idx_preview + 1\n",
+ " )\n",
+ "\n",
+ "if pulse_idx_preview + 1 > data.shape[1]:\n",
+ " pulse_idx_preview = 1\n",
+ " if data.shape[1] > 1:\n",
+ " pulse_idx_preview = 2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e5f0d4d8-e32c-4f2c-8469-4ebbfd3f644c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown(\"### RAW and CORRECTED even/odd pulses for one train:\"))\n",
+ "display(Markdown(f\"Train: {tid}\"))\n",
+ "for mod, pdu in zip(karabo_da, db_modules):\n",
+ " fig, ax = plt.subplots(figsize=(20, 20))\n",
+ " raw_data = mod_dcs[mod][\"train_raw_data\"]\n",
+ " corr_data = mod_dcs[mod][\"train_corr_data\"]\n",
+ "\n",
+ " ax.plot(raw_data[odd_pulse], label=f\"Odd Pulse {odd_pulse}\")\n",
+ " if even_pulse:\n",
+ " ax.plot(raw_data[even_pulse], label=f\"Even Pulse {even_pulse}\")\n",
+ "\n",
+ " ax.set_title(f\"RAW module {mod} ({pdu})\")\n",
+ " ax.set_xlabel(\"Strip #\", size=20)\n",
+ " ax.set_ylabel(\"12-bit ADC RAW\", size=20)\n",
+ " plt.xticks(fontsize=20)\n",
+ " plt.yticks(fontsize=20)\n",
+ " ax.legend()\n",
+ " pass\n",
+ "\n",
+ " fig, ax = plt.subplots(figsize=(20, 20))\n",
+ " ax.plot(corr_data[odd_pulse], label=f\"Odd Pulse {odd_pulse}\")\n",
+ " if even_pulse:\n",
+ " ax.plot(corr_data[even_pulse], label=f\"Even Pulse {even_pulse}\")\n",
+ " ax.set_title(f\"CORRECTED module {mod} ({pdu})\")\n",
+ " ax.set_xlabel(\"Strip #\", size=20)\n",
+ " ax.set_ylabel(\"10-bit KeV CORRECTED\", size=20)\n",
+ " plt.xticks(fontsize=20)\n",
+ " plt.yticks(fontsize=20)\n",
+ " ax.legend()\n",
+ " pass\n",
+ "step_timer.done_step(\"Plotting RAW and CORRECTED odd/even pulses.\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "cal4_venv",
+ "language": "python",
+ "name": "cal4_venv"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..18aa31302652bcbef11897c2b665083d08380218
--- /dev/null
+++ b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
@@ -0,0 +1,264 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# GOTTHARD2 Retrieving Constants Pre-correction #\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "Retrieving Required Constants for Offline Calibration of the Gotthard2 Detector"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/FXE/202221/p003225/raw\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/gotthard2\" # the folder to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
+ "run = 50 # run to process, required\n",
+ "\n",
+ "# Parameters used to access raw data.\n",
+ "karabo_id = \"FXE_XAD_G2XES\" # karabo prefix of Gotthard-II devices\n",
+ "karabo_da = [\"GH201\"] # data aggregators\n",
+ "receiver_template = \"RECEIVER\" # receiver template used to read INSTRUMENT keys.\n",
+ "control_template = \"CONTROL\" # control template used to read CONTROL keys.\n",
+ "instrument_source_template = \"{}/DET/{}:daqOutput\" # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'\n",
+ "ctrl_source_template = \"{}/DET/{}\" # template for control source name (filled with karabo_id_control)\n",
+ "karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n",
+ "\n",
+ "# Parameters for calibration database.\n",
+ "use_dir_creation_date = True # use the creation data of the input dir for database queries.\n",
+ "cal_db_interface = \"tcp://max-exfl017:8017#8025\" # the database interface to use.\n",
+ "cal_db_timeout = 180000 # timeout on caldb requests.\n",
+ "overwrite_creation_time = \"2022-06-28 13:00:00.00\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n",
+ "\n",
+ "# Parameters affecting corrected data.\n",
+ "constants_file = \"\"#/gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\" # Retrieve constants from local.\n",
+ "offset_correction = True # apply offset correction. This can be disabled to only apply LUT or apply LUT and gain correction for non-linear differential results.\n",
+ "gain_correction = True # apply gain correction.\n",
+ "\n",
+ "# Parameter conditions.\n",
+ "bias_voltage = -1 # Detector bias voltage, set to -1 to use value in raw file.\n",
+ "exposure_time = -1. # Detector exposure time, set to -1 to use value in raw file.\n",
+ "exposure_period = -1. # Detector exposure period, set to -1 to use value in raw file.\n",
+ "acquisition_rate = 1.1 # Detector acquisition rate (1.1/4.5), set to -1 to use value in raw file.\n",
+ "single_photon = 0 # Detector single photon mode (High/Low CDS), set to -1 to use value in raw file.\n",
+ "\n",
+ "if constants_file:\n",
+ " print(\"Skipping constant retrieval. Specified constants_file is used.\")\n",
+ " import sys\n",
+ "\n",
+ " sys.exit(0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "from functools import partial\n",
+ "\n",
+ "import multiprocessing\n",
+ "from extra_data import RunDirectory\n",
+ "from pathlib import Path\n",
+ "\n",
+ "from cal_tools.gotthard2 import gotthard2lib\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_from_db,\n",
+ " CalibrationMetadata,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = Path(in_folder)\n",
+ "run_folder = in_folder / f\"r{run:04d}\"\n",
+ "out_folder = Path(out_folder)\n",
+ "out_folder.mkdir(parents=True, exist_ok=True)\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "\n",
+ "if not karabo_id_control:\n",
+ " karabo_id_control = karabo_id\n",
+ "\n",
+ "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
+ "ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)\n",
+ "\n",
+ "print(f\"Retrieve constants for modules: {karabo_da} for run {run}\")\n",
+ "\n",
+ "creation_time = None\n",
+ "if overwrite_creation_time:\n",
+ " creation_time = datetime.datetime.strptime(\n",
+ " overwrite_creation_time, \"%Y-%m-%d %H:%M:%S.%f\"\n",
+ " )\n",
+ "elif use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run)\n",
+ " print(f\"Using {creation_time} as creation time\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Read slow data\n",
+ "run_dc = RunDirectory(run_folder)\n",
+ "g2ctrl = gotthard2lib.Gotthard2Ctrl(run_dc=run_dc, ctrl_src=ctrl_src)\n",
+ "\n",
+ "if bias_voltage == -1:\n",
+ " bias_voltage = g2ctrl.get_bias_voltage()\n",
+ "if exposure_time == -1:\n",
+ " exposure_time = g2ctrl.get_exposure_time()\n",
+ "if exposure_period == -1:\n",
+ " exposure_period = g2ctrl.get_exposure_period()\n",
+ "if acquisition_rate == -1:\n",
+ " acquisition_rate = g2ctrl.get_acquisition_rate()\n",
+ "if single_photon == -1:\n",
+ " single_photon = g2ctrl.get_single_photon()\n",
+ "\n",
+ "print(\"Bias Voltage:\", bias_voltage)\n",
+ "print(\"Exposure Time:\", exposure_time)\n",
+ "print(\"Exposure Period:\", exposure_period)\n",
+ "print(\"Acquisition Rate:\", acquisition_rate)\n",
+ "print(\"Single Photon:\", single_photon)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "condition = Conditions.Dark.Gotthard2(\n",
+ " bias_voltage=bias_voltage,\n",
+ " exposure_time=exposure_time,\n",
+ " exposure_period=exposure_period,\n",
+ " single_photon=single_photon,\n",
+ " acquisition_rate=acquisition_rate,\n",
+ ")\n",
+ "\n",
+ "def get_constants_for_module(mod: str):\n",
+ " \"\"\"Get calibration constants for given module for Gotthard2.\"\"\"\n",
+ " retrieval_function = partial(\n",
+ " get_from_db,\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=mod,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout,\n",
+ " verbosity=1,\n",
+ " meta_only=True,\n",
+ " load_data=False,\n",
+ " empty_constant=None\n",
+ " )\n",
+ "\n",
+ " mdata_dict = dict()\n",
+ " mdata_dict[\"constants\"] = dict()\n",
+ " constants = [\n",
+ " \"LUT\", \"Offset\", \"BadPixelsDark\",\n",
+ " \"RelativeGain\", \"BadPixelsFF\",\n",
+ " ]\n",
+ " for cname in constants:\n",
+ " mdata_dict[\"constants\"][cname] = dict()\n",
+ " if not gain_correction and cname in [\"BadPixelsFF\", \"RelativeGain\"]:\n",
+ " continue\n",
+ " _, mdata = retrieval_function(\n",
+ " condition=condition,\n",
+ " constant=getattr(Constants.Gotthard2, cname)(),\n",
+ " )\n",
+ " mdata_const = mdata.calibration_constant_version\n",
+ " const_mdata = mdata_dict[\"constants\"][cname]\n",
+ " # check if constant was successfully retrieved.\n",
+ " if mdata.comm_db_success:\n",
+ " const_mdata[\"file-path\"] = (\n",
+ " f\"{mdata_const.hdf5path}\" f\"{mdata_const.filename}\"\n",
+ " )\n",
+ " const_mdata[\"dataset-name\"] = mdata_const.h5path\n",
+ " const_mdata[\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " mdata_dict[\"physical-detector-unit\"] = mdata_const.device_name\n",
+ " else:\n",
+ " const_mdata[\"file-path\"] = None\n",
+ " const_mdata[\"creation-time\"] = None\n",
+ " return mdata_dict, mod\n",
+ "\n",
+ "with multiprocessing.Pool() as pool:\n",
+ " results = pool.map(get_constants_for_module, karabo_da)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n",
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
+ "timestamps = dict()\n",
+ "\n",
+ "for md_dict, mod in results:\n",
+ " if mod in retrieved_constants:\n",
+ " print(f\"Constant for {mod} already in calibration_metadata.yml, won't query again.\") # noqa\n",
+ " continue\n",
+ " retrieved_constants[mod] = md_dict\n",
+ " module_timestamps = timestamps[mod] = dict()\n",
+ "\n",
+ " print(f\"Module: {mod}:\")\n",
+ " for cname, mdata in md_dict[\"constants\"].items():\n",
+ " if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
+ " mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
+ " print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
+ "\n",
+ " for cname in [\"Offset\", \"BadPixelsDark\", \"RelativeGain\", \"BadPixelsFF\"]:\n",
+ " if cname in md_dict[\"constants\"]:\n",
+ " module_timestamps[cname] = md_dict[\"constants\"][cname][\"creation-time\"]\n",
+ " else:\n",
+ " module_timestamps[cname] = \"NA\"\n",
+ "\n",
+ "time_summary = retrieved_constants.setdefault(\"time-summary\", {})\n",
+ "time_summary[\"SAll\"] = timestamps\n",
+ "\n",
+ "metadata.save()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.8.11 ('.cal4_venv')",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ },
+ "orig_nbformat": 4,
+ "vscode": {
+ "interpreter": {
+ "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index f9f57a7943b5e8ffcf7ee5ebb3b94c5ff240c27b..2a4aba6902a03d2e94f799edc318d6da66d3319e 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -19,7 +19,8 @@
"source": [
"in_folder = \"/gpfs/exfel/exp/SPB/202130/p900204/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove\" # the folder to output to, required\n",
- "run = 112 # run to process, required\n",
+ "run = 91 # run to process, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequences = [-1] # sequences to correct, set to [-1] for all, range allowed\n",
"sequences_per_node = 1 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel\n",
"\n",
@@ -37,23 +38,24 @@
"cal_db_timeout = 180000 # timeout on caldb requests\n",
"\n",
"# Parameters affecting corrected data.\n",
- "overwrite = True # set to True if existing data should be overwritten\n",
"relative_gain = True # do relative gain correction\n",
- "plt_images = 100 # Number of images to plot after applying selected corrections.\n",
"limit_images = 0 # ONLY FOR TESTING. process only first N images, Use 0 to process all.\n",
- "cell_id_preview = 15 # cell Id used for preview in single-shot plots\n",
"\n",
"# Parameters for retrieving calibration constants\n",
- "manual_slow_data = False # if true, use manually entered bias_voltage and integration_time values\n",
+ "manual_slow_data = False # if true, use manually entered bias_voltage, integration_time, gain_setting, and gain_mode values\n",
"integration_time = 4.96 # integration time in us, will be overwritten by value in file\n",
"gain_setting = 0 # 0 for dynamic gain, 1 for dynamic HG0, will be overwritten by value in file\n",
"gain_mode = 0 # 0 for runs with dynamic gain setting, 1 for fixgain. It will be overwritten by value in file, if manual_slow_data is set to True.\n",
- "mem_cells = 0 # leave memory cells equal 0, as it is saved in control information starting 2019.\n",
+ "mem_cells = -1 # Set mem_cells to -1 to automatically use the value stored in RAW data.\n",
"bias_voltage = 180 # will be overwritten by value in file\n",
"\n",
"# Parameters for plotting\n",
"skip_plots = False # exit after writing corrected files\n",
+ "plot_trains = 500 # Number of trains to plot for RAW and CORRECTED plots. Set to -1 to automatically plot all trains.\n",
+ "cell_id_preview = 15 # cell Id used for preview in single-shot plots\n",
"\n",
+ "# Parameters for ROI selection and reduction\n",
+ "roi_definitions = [-1] # List with groups of 6 values defining ROIs, e.g. [3, 120, 180, 200, 550, -2] for module 3 (JNGFR03), slice 120:180, 200:550, average along axis -2 (slow scan, or -1 for fast scan)\n",
"\n",
"def balance_sequences(in_folder, run, sequences, sequences_per_node, karabo_da):\n",
" from xfel_calibrate.calibrate import balance_sequences as bs\n",
@@ -67,6 +69,7 @@
"outputs": [],
"source": [
"import multiprocessing\n",
+ "import sys\n",
"import warnings\n",
"from functools import partial\n",
"from pathlib import Path\n",
@@ -78,7 +81,8 @@
"import pasha as psh\n",
"import tabulate\n",
"from IPython.display import Latex, Markdown, display\n",
- "from extra_data import H5File, RunDirectory\n",
+ "from extra_data import H5File, RunDirectory, by_id, components\n",
+ "from extra_geom import JUNGFRAUGeometry\n",
"from matplotlib.colors import LogNorm\n",
"\n",
"from cal_tools import h5_copy_except\n",
@@ -91,6 +95,7 @@
" get_pdu_from_db,\n",
" map_seq_files,\n",
" write_compressed_frames,\n",
+ " CalibrationMetadata,\n",
")\n",
"from iCalibrationDB import Conditions, Constants\n",
"\n",
@@ -108,13 +113,11 @@
"source": [
"in_folder = Path(in_folder)\n",
"out_folder = Path(out_folder)\n",
- "run_dc = RunDirectory(in_folder / f'r{run:04d}')\n",
+ "run_folder = in_folder / f'r{run:04d}'\n",
+ "run_dc = RunDirectory(run_folder)\n",
"instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
"\n",
- "if out_folder.exists() and not overwrite:\n",
- " raise AttributeError(\"Output path exists! Exiting\")\n",
- "else:\n",
- " out_folder.mkdir(parents=True, exist_ok=True)\n",
+ "out_folder.mkdir(parents=True, exist_ok=True)\n",
"\n",
"print(f\"Run is: {run}\")\n",
"print(f\"Instrument H5File source: {instrument_src}\")\n",
@@ -126,7 +129,12 @@
" print(f\"Using {creation_time} as creation time\")\n",
"\n",
"if karabo_id_control == \"\":\n",
- " karabo_id_control = karabo_id"
+ " karabo_id_control = karabo_id\n",
+ " \n",
+ "if any(axis_no not in {-2, -1, 2, 3} for axis_no in roi_definitions[5::6]):\n",
+ " print(\"ROI averaging must be on axis 2/3 (or equivalently -2/-1). \"\n",
+ " f\"Axis numbers given: {roi_definitions[5::6]}\")\n",
+ " sys.exit(1)"
]
},
{
@@ -137,7 +145,7 @@
"source": [
"# Read available sequence files to correct.\n",
"mapped_files, num_seq_files = map_seq_files(\n",
- " run_dc, karabo_id, karabo_da, sequences)\n",
+ " run_folder, karabo_da, sequences)\n",
"\n",
"if not len(mapped_files):\n",
" raise IndexError(\n",
@@ -173,25 +181,16 @@
"source": [
"ctrl_src = ctrl_source_template.format(karabo_id_control)\n",
"ctrl_data = JungfrauCtrl(run_dc, ctrl_src)\n",
- "try:\n",
- " this_run_mcells, sc_start = ctrl_data.get_memory_cells()\n",
"\n",
- " if this_run_mcells == 1:\n",
- " memory_cells = 1\n",
- " print(\"Run is in single cell mode.\\n\"\n",
- " f\"Storage cell start: {sc_start:02d}\")\n",
- " else:\n",
- " memory_cells = 16\n",
- " print(f\"Run is in burst mode.\\n\"\n",
- " f\"Storage cell start: {sc_start:02d}\")\n",
- "except KeyError as e:\n",
- " print(\"WARNING: KeyError while reading number of memory cells.\")\n",
- " if mem_cells == 0:\n",
- " memory_cells = 1\n",
- " else:\n",
- " memory_cells = mem_cells\n",
- " print(f\"WARNING: Set memory cells to {memory_cells}, as \"\n",
- " \"it is not saved in control information.\")\n",
+ "if mem_cells < 0:\n",
+ " memory_cells, sc_start = ctrl_data.get_memory_cells()\n",
+ "\n",
+ " mem_cells_name = \"single cell\" if memory_cells == 1 else \"burst\"\n",
+ " print(f\"Run is in {mem_cells_name} mode.\\nStorage cell start: {sc_start:02d}\")\n",
+ "else:\n",
+ " memory_cells = mem_cells\n",
+ " mem_cells_name = \"single cell\" if memory_cells == 1 else \"burst\"\n",
+ " print(f\"Run is in manually set to {mem_cells_name} mode. With {memory_cells} memory cells\")\n",
"\n",
"if not manual_slow_data:\n",
" integration_time = ctrl_data.get_integration_time()\n",
@@ -228,6 +227,16 @@
" gain_mode=gain_mode,\n",
")\n",
"\n",
+ "empty_constants = {\n",
+ " \"Offset\": np.zeros((512, 1024, memory_cells, 3), dtype=np.float32),\n",
+ " \"BadPixelsDark\": np.zeros((512, 1024, memory_cells, 3), dtype=np.uint32),\n",
+ " \"RelativeGain\": None,\n",
+ " \"BadPixelsFF\": None,\n",
+ "}\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# NOTE: this notebook will not overwrite calibration metadata file\n",
+ "const_yaml = metadata.get(\"retrieved-constants\", {})\n",
+ "\n",
"def get_constants_for_module(karabo_da: str):\n",
" \"\"\" Get calibration constants for given module of Jungfrau\n",
"\n",
@@ -236,41 +245,46 @@
" mask (mask of bad pixels),\n",
" gain_map (map of relative gain factors),\n",
" db_module (name of DB module),\n",
- " when (dictionaty: constant - creation time)\n",
+ " when (dictionary: constant - creation time)\n",
" \"\"\"\n",
"\n",
- " when = {}\n",
- " retrieval_function = partial(\n",
- " get_constant_from_db_and_time,\n",
- " karabo_id=karabo_id,\n",
- " karabo_da=karabo_da,\n",
- " cal_db_interface=cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " timeout=cal_db_timeout,\n",
- " print_once=False,\n",
- " )\n",
- " offset_map, when[\"Offset\"] = retrieval_function(\n",
- " condition=condition,\n",
- " constant=Constants.jungfrau.Offset(),\n",
- " empty_constant=np.zeros((512, 1024, memory_cells, 3))\n",
- " )\n",
- " mask, when[\"BadPixelsDark\"] = retrieval_function(\n",
- " condition=condition,\n",
- " constant=Constants.jungfrau.BadPixelsDark(),\n",
- " empty_constant=np.zeros((512, 1024, memory_cells, 3), dtype=np.uint32),\n",
- " )\n",
- " mask_ff, when[\"BadPixelsFF\"] = retrieval_function(\n",
- " condition=condition,\n",
- " constant=Constants.jungfrau.BadPixelsFF(),\n",
- " empty_constant=None\n",
- " )\n",
- " gain_map, when[\"Gain\"] = retrieval_function(\n",
- " condition=condition,\n",
- " constant=Constants.jungfrau.RelativeGain(),\n",
- " empty_constant=None\n",
- " )\n",
+ " when = dict()\n",
+ " const_data = dict()\n",
+ "\n",
+ " if const_yaml:\n",
+ " for cname, mdata in const_yaml[karabo_da][\"constants\"].items():\n",
+ " const_data[cname] = dict()\n",
+ " when[cname] = mdata[\"creation-time\"]\n",
+ " if when[cname]:\n",
+ " with h5py.File(mdata[\"file-path\"], \"r\") as cf:\n",
+ " const_data[cname] = np.copy(\n",
+ " cf[f\"{mdata['dataset-name']}/data\"])\n",
+ " else:\n",
+ " const_data[cname] = empty_constants[cname]\n",
+ " else:\n",
+ " retrieval_function = partial(\n",
+ " get_constant_from_db_and_time,\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout,\n",
+ " print_once=False,\n",
+ " )\n",
+ " \n",
+ " for cname, cempty in empty_constants.items():\n",
+ " const_data[cname], when[cname] = retrieval_function(\n",
+ " condition=condition,\n",
+ " constant=getattr(Constants.jungfrau, cname)(),\n",
+ " empty_constant=cempty,\n",
+ " )\n",
+ "\n",
+ " offset_map = const_data[\"Offset\"]\n",
+ " mask = const_data[\"BadPixelsDark\"]\n",
+ " gain_map = const_data[\"RelativeGain\"]\n",
+ " mask_ff = const_data[\"BadPixelsFF\"]\n",
"\n",
- " # combine masks\n",
+ " # Combine masks\n",
" if mask_ff is not None:\n",
" mask |= np.moveaxis(mask_ff, 0, 1)\n",
"\n",
@@ -281,6 +295,10 @@
" else:\n",
" offset_map = np.squeeze(offset_map)\n",
" mask = np.squeeze(mask)\n",
+ " \n",
+ " # masking double size pixels\n",
+ " mask[..., [255, 256], :, :] |= BadPixels.NON_STANDARD_SIZE\n",
+ " mask[..., [255, 256, 511, 512, 767, 768], :] |= BadPixels.NON_STANDARD_SIZE\n",
"\n",
" if gain_map is not None:\n",
" if memory_cells > 1:\n",
@@ -377,6 +395,63 @@
"print(f\"Using {n_cpus} workers for correction.\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def save_reduced_rois(ofile, data_corr, mask_corr, karabo_da):\n",
+ " \"\"\"If ROIs are defined for this karabo_da, reduce them and save to the output file\"\"\"\n",
+ " rois_defined = 0\n",
+ " module_no = int(karabo_da[-2:])\n",
+ " params_source = f'{karabo_id}/ROIPROC/{karabo_da}'\n",
+ " rois_source = f'{params_source}:output/data'\n",
+ " \n",
+ " for i in range(len(roi_definitions) // 6):\n",
+ " roi_module, a1, a2, b1, b2, mean_axis = roi_definitions[i*6 : (i+1)*6]\n",
+ " if roi_module == module_no:\n",
+ " rois_defined += 1\n",
+ " # Apply the mask and average remaining pixels to 1D\n",
+ " roi_data = data_corr[..., a1:a2, b1:b2].mean(\n",
+ " axis=mean_axis, where=(mask_corr[..., a1:a2, b1:b2] == 0)\n",
+ " )\n",
+ " ofile.create_dataset(\n",
+ " f'INSTRUMENT/{rois_source}/roi{rois_defined}/data',\n",
+ " data=roi_data\n",
+ " )\n",
+ " ofile.require_group(f'CONTROL/{params_source}')\n",
+ " params_grp = ofile.create_group(f'RUN/{params_source}/roi{rois_defined}')\n",
+ " params_grp['region'] = np.array([[a1, a2, b1, b2]])\n",
+ " params_grp['reduce_axis'] = np.array([mean_axis])\n",
+ " \n",
+ " if rois_defined:\n",
+ " # Copy the index for the new source\n",
+ " ofile.copy(f'INDEX/{karabo_id}/DET/{karabo_da}:daqOutput/data',\n",
+ " f'INDEX/{rois_source}')\n",
+ " ntrains = ofile['INDEX/trainId'].shape[0]\n",
+ " ofile.create_dataset(f'INDEX/{params_source}/count', shape=(ntrains,), dtype=np.uint64)\n",
+ " ofile.create_dataset(f'INDEX/{params_source}/first', shape=(ntrains,), dtype=np.uint64)\n",
+ " \n",
+ " # Add the new source to the list in METADATA\n",
+ " if 'dataSourceId' in ofile['METADATA']:\n",
+ " # Older file format\n",
+ " data_sources_grp = ofile['METADATA'] \n",
+ " else:\n",
+ " # Newer file format\n",
+ " data_sources_grp = ofile['METADATA/dataSources']\n",
+ " \n",
+ " def extend(dset, values):\n",
+ " dset.resize(dset.shape[0] + len(values), axis=0)\n",
+ " dset[-len(values):] = values\n",
+ " \n",
+ " extend(data_sources_grp['root'], ['CONTROL', 'INSTRUMENT'])\n",
+ " extend(data_sources_grp['deviceId'], [params_source, rois_source])\n",
+ " extend(data_sources_grp['dataSourceId'], [\n",
+ " f'CONTROL/{params_source}', f'INSTRUMENT/{rois_source}']\n",
+ " )\n"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -390,11 +465,6 @@
"metadata": {},
"outputs": [],
"source": [
- "fim_data = {}\n",
- "gim_data = {}\n",
- "rim_data = {}\n",
- "msk_data = {}\n",
- "\n",
"# Loop over modules\n",
"for local_karabo_da, mapped_files_module in mapped_files.items():\n",
" instrument_src_kda = instrument_src.format(int(local_karabo_da[-2:]))\n",
@@ -431,13 +501,10 @@
" print(f\"\\t- WARNING: {sequence_file.name} has {n_trains - dshape[0]} \"\n",
" \"trains with empty data.\")\n",
"\n",
- " # Just in case if n_imgs is less than the chosen plt_images.\n",
- " plt_images = min(plt_images, n_imgs)\n",
- "\n",
" # load constants from the constants dictionary.\n",
" offset_map, mask, gain_map = constants[local_karabo_da]\n",
"\n",
- " # Allocate shared arrays.\n",
+ " # Allocate shared arrays for corrected data.\n",
" data_corr = context.alloc(shape=dshape, dtype=np.float32)\n",
" mask_corr = context.alloc(shape=dshape, dtype=np.uint32)\n",
"\n",
@@ -461,8 +528,6 @@
" else:\n",
" cell_idx_preview = 0\n",
"\n",
- " rim_data[local_karabo_da] = data[:plt_images, cell_idx_preview, ...].copy()\n",
- "\n",
" context.map(correct_train, data)\n",
" step_timer.done_step(f'Correction time.')\n",
"\n",
@@ -489,17 +554,9 @@
" dataset_path=f\"{data_path}/mask\",\n",
" comp_threads=n_cpus,\n",
" )\n",
+ " save_reduced_rois(ofile, data_corr, mask_corr, local_karabo_da)\n",
"\n",
- " step_timer.done_step(f'Saving data time.')\n",
- "\n",
- " # Prepare plotting arrays\n",
- " step_timer.start()\n",
- "\n",
- " fim_data[local_karabo_da] = data_corr[:plt_images, cell_idx_preview, ...].copy()\n",
- " msk_data[local_karabo_da] = mask_corr[:plt_images, cell_idx_preview, ...].copy()\n",
- " gim_data[local_karabo_da] = gain[:plt_images, cell_idx_preview, ...].copy()\n",
- "\n",
- " step_timer.done_step(f'Preparing plotting data of {plt_images} images.')"
+ " step_timer.done_step(f'Saving data time.')"
]
},
{
@@ -537,27 +594,75 @@
"metadata": {},
"outputs": [],
"source": [
- "def do_2d_plot(data, edges, y_axis, x_axis, title):\n",
- " fig = plt.figure(figsize=(10, 10))\n",
- " ax = fig.add_subplot(111)\n",
- " extent = [\n",
- " np.min(edges[1]),\n",
- " np.max(edges[1]),\n",
- " np.min(edges[0]),\n",
- " np.max(edges[0]),\n",
+ "# Positions are given in pixels\n",
+ "mod_width = (256 * 4) + (2 * 3) # inc. 2px gaps between tiles\n",
+ "mod_height = (256 * 2) + 2\n",
+ "if karabo_id == \"SPB_IRDA_JF4M\":\n",
+ " # The first 4 modules are rotated 180 degrees relative to the others.\n",
+ " # We pass the bottom, beam-right corner of the module regardless of its\n",
+ " # orientation, requiring a subtraction from the symmetric positions we'd\n",
+ " # otherwise calculate.\n",
+ " x_start, y_start = 1125, 1078\n",
+ " module_pos = [\n",
+ " (x_start - mod_width, y_start - mod_height - (i * (mod_height + 33)))\n",
+ " for i in range(4)\n",
+ " ] + [\n",
+ " (-x_start, -y_start + (i * (mod_height + 33))) for i in range(4)\n",
" ]\n",
+ " orientations = [(-1, -1) for _ in range(4)] + [(1, 1) for _ in range(4)]\n",
+ "elif karabo_id == \"FXE_XAD_JF1M\":\n",
+ " module_pos = ((-mod_width//2, 33),(-mod_width//2, -mod_height -33))\n",
+ " orientations = [(-1,-1), (1,1)]\n",
+ "else:\n",
+ " module_pos = ((-mod_width//2,-mod_height//2),)\n",
+ " orientations = None\n",
"\n",
- " im = ax.imshow(\n",
- " data[::-1, :],\n",
- " extent=extent,\n",
- " aspect=\"auto\",\n",
- " norm=LogNorm(vmin=1, vmax=np.max(data))\n",
+ "geom = JUNGFRAUGeometry.from_module_positions(module_pos, orientations=orientations, asic_gap=0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "first_seq = 0 if sequences == [-1] else sequences[0]\n",
+ "\n",
+ "with RunDirectory(out_folder, f\"*{run}*S{first_seq:05d}*\") as corr_dc:\n",
+ " # Reading CORR data for plotting.\n",
+ " jf_corr = components.JUNGFRAU(\n",
+ " corr_dc,\n",
+ " detector_name=karabo_id,\n",
+ " ).select_trains(np.s_[:plot_trains])\n",
+ " tid, jf_corr_data = next(iter(jf_corr.trains(require_all=True)))\n",
+ "\n",
+ "# Shape = [modules, trains, cells, x, y]\n",
+ "corrected = jf_corr.get_array(\"data.adc\")[:, :, cell_idx_preview, ...].values\n",
+ "corrected_train = jf_corr_data[\"data.adc\"][\n",
+ " :, cell_idx_preview, ...\n",
+ "].values # loose the train axis.\n",
+ "\n",
+ "mask = jf_corr.get_array(\"data.mask\")[:, :, cell_idx_preview, ...].values\n",
+ "mask_train = jf_corr_data[\"data.mask\"][:, cell_idx_preview, ...].values\n",
+ "\n",
+ "with RunDirectory(f\"{in_folder}/r{run:04d}/\", f\"*S{first_seq:05d}*\") as raw_dc:\n",
+ "\n",
+ " # Reading RAW data for plotting.\n",
+ " jf_raw = components.JUNGFRAU(raw_dc, detector_name=karabo_id).select_trains(\n",
+ " np.s_[:plot_trains]\n",
" )\n",
- " ax.set_xlabel(x_axis)\n",
- " ax.set_ylabel(y_axis)\n",
- " ax.set_title(title)\n",
- " cb = fig.colorbar(im)\n",
- " cb.set_label(\"Counts\")"
+ "\n",
+ "raw = jf_raw.get_array(\"data.adc\")[:, :, cell_idx_preview, ...].values\n",
+ "raw_train = (\n",
+ " jf_raw.select_trains(by_id[[tid]])\n",
+ " .get_array(\"data.adc\")[:, 0, cell_idx_preview, ...]\n",
+ " .values\n",
+ ")\n",
+ "\n",
+ "gain = jf_raw.get_array(\"data.gain\")[:, :, cell_idx_preview, ...].values\n",
+ "gain_train_cells = (\n",
+ " jf_raw.select_trains(by_id[[tid]]).get_array(\"data.gain\")[:, :, :, ...].values\n",
+ ")"
]
},
{
@@ -572,7 +677,15 @@
" constant=Constants.jungfrau.Offset(),\n",
" condition=condition,\n",
" cal_db_interface=cal_db_interface,\n",
- " snapshot_at=creation_time)"
+ " snapshot_at=creation_time,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Mean RAW Preview"
]
},
{
@@ -581,27 +694,27 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " h, ex, ey = np.histogram2d(\n",
- " rim_data[mod].flatten(),\n",
- " gim_data[mod].flatten(),\n",
- " bins=[100, 4],\n",
- " range=[[0, 10000], [0, 4]],\n",
- " )\n",
- " do_2d_plot(\n",
- " h, (ex, ey),\n",
- " \"Signal (ADU)\",\n",
- " \"Gain Bit Value (high gain=0[00], medium gain=1[01], low gain=3[11])\",\n",
- " f\"Module {mod} ({pdu})\")"
+ "print(f\"The per pixel mean of the first {raw.shape[1]} trains of the first sequence file\")\n",
+ "\n",
+ "fig, ax = plt.subplots(figsize=(18, 10))\n",
+ "raw_mean = np.mean(raw, axis=1)\n",
+ "geom.plot_data_fast(\n",
+ " raw_mean,\n",
+ " ax=ax,\n",
+ " vmin=min(0.75*np.median(raw_mean[raw_mean > 0]), 2000),\n",
+ " vmax=max(1.5*np.median(raw_mean[raw_mean > 0]), 16000),\n",
+ " cmap=\"jet\",\n",
+ " colorbar={'shrink': 1, 'pad': 0.01},\n",
+ ")\n",
+ "ax.set_title(f'{karabo_id} - Mean RAW', size=18)\n",
+ "plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "### Mean RAW Preview ###\n",
- "\n",
- "The per pixel mean of the sequence file of RAW data"
+ "### Mean CORRECTED Preview"
]
},
{
@@ -610,27 +723,69 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
- " ax = fig.add_subplot(111)\n",
+ "print(f\"The per pixel mean of the first {corrected.shape[1]} trains of the first sequence file\")\n",
+ "\n",
+ "fig, ax = plt.subplots(figsize=(18, 10))\n",
+ "corrected_mean = np.mean(corrected, axis=1)\n",
+ "_corrected_vmin = min(0.75*np.median(corrected_mean[corrected_mean > 0]), -0.5)\n",
+ "_corrected_vmax = max(2.*np.median(corrected_mean[corrected_mean > 0]), 100)\n",
+ "geom.plot_data_fast(\n",
+ " corrected_mean,\n",
+ " ax=ax,\n",
+ " vmin=_corrected_vmin,\n",
+ " vmax=_corrected_vmax,\n",
+ " cmap=\"jet\",\n",
+ " colorbar={'shrink': 1, 'pad': 0.01},\n",
+ ")\n",
+ "ax.set_title(f'{karabo_id} - Mean CORRECTED', size=18)\n",
"\n",
- " im = ax.imshow(\n",
- " np.mean(rim_data[mod],axis=0),\n",
- " vmin=min(0.75*np.median(rim_data[mod][rim_data[mod] > 0]), 2000),\n",
- " vmax=max(1.5*np.median(rim_data[mod][rim_data[mod] > 0]), 16000),\n",
- " cmap=\"jet\")\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fig, ax = plt.subplots(figsize=(18, 10))\n",
+ "corrected_masked = corrected.copy()\n",
+ "corrected_masked[mask != 0] = np.nan\n",
+ "corrected_masked_mean = np.nanmean(corrected_masked, axis=1)\n",
+ "del corrected_masked\n",
+ "geom.plot_data_fast(\n",
+ " corrected_masked_mean,\n",
+ " ax=ax,\n",
+ " vmin=_corrected_vmin,\n",
+ " vmax=_corrected_vmax,\n",
+ " cmap=\"jet\",\n",
+ " colorbar={'shrink': 1, 'pad': 0.01},\n",
+ ")\n",
+ "ax.set_title(f'{karabo_id} - Mean CORRECTED with mask', size=18)\n",
"\n",
- " ax.set_title(f'Module {mod} ({pdu})')\n",
- " cb = fig.colorbar(im, ax=ax)"
+ "plt.show()"
]
},
{
- "cell_type": "markdown",
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
- "### Mean CORRECTED Preview ###\n",
+ "display(Markdown((f\"#### A single image from train {tid}\")))\n",
+ "\n",
+ "fig, ax = plt.subplots(figsize=(18, 10))\n",
+ "geom.plot_data_fast(\n",
+ " corrected_train,\n",
+ " ax=ax,\n",
+ " vmin=min(0.75 * np.median(corrected_train[corrected_train > 0]), -0.5),\n",
+ " vmax=max(2.0 * np.median(corrected_train[corrected_train > 0]), 100),\n",
+ " cmap=\"jet\",\n",
+ " colorbar={\"shrink\": 1, \"pad\": 0.01},\n",
+ ")\n",
+ "ax.set_title(f\"{karabo_id} - CORRECTED train: {tid}\", size=18)\n",
"\n",
- "The per pixel mean of the sequence file of CORR data"
+ "plt.show()"
]
},
{
@@ -639,27 +794,34 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
+ "def do_2d_plot(data, edges, y_axis, x_axis, title):\n",
+ " fig = plt.figure(figsize=(10, 10))\n",
" ax = fig.add_subplot(111)\n",
+ " extent = [\n",
+ " np.min(edges[1]),\n",
+ " np.max(edges[1]),\n",
+ " np.min(edges[0]),\n",
+ " np.max(edges[0]),\n",
+ " ]\n",
"\n",
" im = ax.imshow(\n",
- " np.mean(fim_data[mod],axis=0),\n",
- " vmin=min(0.75*np.median(fim_data[mod][fim_data[mod] > 0]), -0.5),\n",
- " vmax=max(2.*np.median(fim_data[mod][fim_data[mod] > 0]), 100),\n",
- " cmap=\"jet\")\n",
- "\n",
- " ax.set_title(f'Module {mod} ({pdu})', size=18)\n",
- " cb = fig.colorbar(im, ax=ax)"
+ " data[::-1, :],\n",
+ " extent=extent,\n",
+ " aspect=\"auto\",\n",
+ " norm=LogNorm(vmin=1, vmax=np.max(data))\n",
+ " )\n",
+ " ax.set_xlabel(x_axis)\n",
+ " ax.set_ylabel(y_axis)\n",
+ " ax.set_title(title)\n",
+ " cb = fig.colorbar(im)\n",
+ " cb.set_label(\"Counts\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "### Single Train Preview ###\n",
- "\n",
- "A single image from the first train"
+ "### Gain Bit Value"
]
},
{
@@ -668,18 +830,20 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
- " ax = fig.add_subplot(111)\n",
- "\n",
- " im = ax.imshow(\n",
- " fim_data[mod][0, ...],\n",
- " vmin=min(0.75*np.median(fim_data[mod][0, ...]), -0.5),\n",
- " vmax=max(2.*np.median(fim_data[mod][0, ...]), 100),\n",
- " cmap=\"jet\")\n",
- "\n",
- " ax.set_title(f'Module {mod} ({pdu})', size=18)\n",
- " cb = fig.colorbar(im, ax=ax)"
+ "for i, (pdu, mod) in enumerate(zip(db_modules, karabo_da)):\n",
+ " h, ex, ey = np.histogram2d(\n",
+ " raw[i].flatten(),\n",
+ " gain[i].flatten(),\n",
+ " bins=[100, 4],\n",
+ " range=[[0, 10000], [0, 4]],\n",
+ " )\n",
+ " do_2d_plot(\n",
+ " h,\n",
+ " (ex, ey),\n",
+ " \"Signal (ADU)\",\n",
+ " \"Gain Bit Value (high gain=0[00], medium gain=1[01], low gain=3[11])\",\n",
+ " f\"Module {mod} ({pdu})\",\n",
+ " )"
]
},
{
@@ -695,38 +859,26 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20,10))\n",
- " ax = fig.add_subplot(211)\n",
- " h = ax.hist(\n",
- " fim_data[mod].flatten(),\n",
- " bins=1000,\n",
- " range=(-100, 1000),\n",
- " log=True,\n",
- " )\n",
- " l = ax.set_xlabel(\"Signal (keV)\")\n",
- " l = ax.set_ylabel(\"Counts\")\n",
- " _ = ax.set_title(f'Module {mod} ({pdu})')\n",
- "\n",
- " ax = fig.add_subplot(212)\n",
- " h = ax.hist(\n",
- " fim_data[mod].flatten(),\n",
- " bins=1000,\n",
- " range=(-1000, 10000),\n",
- " log=True,\n",
- " )\n",
- " l = ax.set_xlabel(\"Signal (keV)\")\n",
- " l = ax.set_ylabel(\"Counts\")\n",
- " _ = ax.set_title(f'Module {mod} ({pdu})')"
+ "for i, (pdu, mod) in enumerate(zip(db_modules, karabo_da)): \n",
+ " fig, axs = plt.subplots(nrows=2, ncols=1, figsize=(18, 10))\n",
+ " corrected_flatten = corrected[i].flatten()\n",
+ " for ax, hist_range in zip(axs, [(-100, 1000), (-1000, 10000)]):\n",
+ " h = ax.hist(\n",
+ " corrected_flatten,\n",
+ " bins=1000,\n",
+ " range=hist_range,\n",
+ " log=True,\n",
+ " )\n",
+ " l = ax.set_xlabel(\"Signal (keV)\")\n",
+ " l = ax.set_ylabel(\"Counts\")\n",
+ " _ = ax.set_title(f'Module {mod} ({pdu})')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "### Maximum GAIN Preview ###\n",
- "\n",
- "The per pixel maximum of the first train of the GAIN data"
+ "### Maximum GAIN Preview"
]
},
{
@@ -735,14 +887,17 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
- " ax = fig.add_subplot(111)\n",
- " im = ax.imshow(\n",
- " np.max(gim_data[mod], axis=0),\n",
- " vmin=0, vmax=3, cmap=\"jet\")\n",
- " ax.set_title(f'Module {mod} ({pdu})', size=18)\n",
- " cb = fig.colorbar(im, ax=ax)"
+ "display(Markdown((f\"#### The per pixel maximum of train {tid} of the GAIN data\")))\n",
+ "\n",
+ "fig, ax = plt.subplots(figsize=(18, 10))\n",
+ "gain_max = np.max(gain_train_cells, axis=(1, 2))\n",
+ "geom.plot_data_fast(\n",
+ " gain_max,\n",
+ " ax=ax,\n",
+ " cmap=\"jet\",\n",
+ " colorbar={'shrink': 1, 'pad': 0.01},\n",
+ ")\n",
+ "plt.show()"
]
},
{
@@ -783,14 +938,16 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
- " ax = fig.add_subplot(111)\n",
- " im = ax.imshow(\n",
- " np.log2(msk_data[mod][0,...]),\n",
- " vmin=0, vmax=32, cmap=\"jet\")\n",
- " ax.set_title(f'Module {mod} ({pdu})', size=18)\n",
- " cb = fig.colorbar(im, ax=ax)"
+ "display(Markdown(f\"#### Bad pixels image for train {tid}\"))\n",
+ "\n",
+ "fig, ax = plt.subplots(figsize=(18, 10))\n",
+ "geom.plot_data_fast(\n",
+ " np.log2(mask_train),\n",
+ " ax=ax,\n",
+ " vmin=0, vmax=32, cmap=\"jet\",\n",
+ " colorbar={'shrink': 1, 'pad': 0.01},\n",
+ ")\n",
+ "plt.show()"
]
}
],
@@ -810,7 +967,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.11"
+ "version": "3.8.10"
}
},
"nbformat": 4,
diff --git a/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..38db199e791f2b7b86cb6566a850d40edba0f3fe
--- /dev/null
+++ b/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb
@@ -0,0 +1,270 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# JUNGFRAU Retrieving Constants Pre-correction #\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "Retrieving Required Constants for Offline Calibration of the JUNGFRAU Detector"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/SPB/202130/p900204/raw\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove\" # the folder to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
+ "run = 112 # run to process, required\n",
+ "sequences = [-1] # sequences to correct, set to [-1] for all, range allowed\n",
+ "sequences_per_node = 1 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel\n",
+ "\n",
+ "# Parameters used to access raw data.\n",
+ "karabo_id = \"SPB_IRDA_JF4M\" # karabo prefix of Jungfrau devices\n",
+ "karabo_da = ['JNGFR01', 'JNGFR02', 'JNGFR03', 'JNGFR04', 'JNGFR05', 'JNGFR06', 'JNGFR07', 'JNGFR08'] # data aggregators\n",
+ "receiver_template = \"JNGFR{:02d}\" # Detector receiver template for accessing raw data files. e.g. \"JNGFR{:02d}\"\n",
+ "instrument_source_template = '{}/DET/{}:daqOutput' # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'\n",
+ "ctrl_source_template = '{}/DET/CONTROL' # template for control source name (filled with karabo_id_control)\n",
+ "karabo_id_control = \"\" # if control is on a different ID, set to empty string if it is the same a karabo-id\n",
+ "\n",
+ "# Parameters for calibration database.\n",
+ "use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
+ "cal_db_interface = \"tcp://max-exfl016:8017#8025\" # the database interface to use\n",
+ "cal_db_timeout = 180000 # timeout on cal db requests\n",
+ "\n",
+ "# Parameters affecting corrected data.\n",
+ "relative_gain = True # do relative gain correction\n",
+ "\n",
+ "# Parameters for retrieving calibration constants\n",
+ "manual_slow_data = False # if true, use manually entered bias_voltage, integration_time, gain_setting, and gain_mode values\n",
+ "integration_time = 4.96 # integration time in us, will be overwritten by value in file\n",
+ "gain_setting = 0 # 0 for dynamic gain, 1 for dynamic HG0, will be overwritten by value in file\n",
+ "gain_mode = 0 # 0 for runs with dynamic gain setting, 1 for fixgain. It will be overwritten by value in file, if manual_slow_data is set to True.\n",
+ "mem_cells = -1 # Set mem_cells to -1 to automatically use the value stored in RAW data.#\n",
+ "bias_voltage = 180 # will be overwritten by value in file"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "from functools import partial\n",
+ "\n",
+ "import multiprocessing\n",
+ "from extra_data import RunDirectory\n",
+ "from pathlib import Path\n",
+ "\n",
+ "from cal_tools.jungfraulib import JungfrauCtrl\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_from_db,\n",
+ " CalibrationMetadata,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = Path(in_folder)\n",
+ "out_folder = Path(out_folder)\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "run_folder = in_folder / f'r{run:04d}'\n",
+ "run_dc = RunDirectory(run_folder)\n",
+ "\n",
+ "out_folder.mkdir(parents=True, exist_ok=True)\n",
+ "\n",
+ "creation_time = None\n",
+ "if use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run)\n",
+ " print(f\"Using {creation_time} as creation time\")\n",
+ "\n",
+ "if karabo_id_control == \"\":\n",
+ " karabo_id_control = karabo_id"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "ctrl_src = ctrl_source_template.format(karabo_id_control)\n",
+ "ctrl_data = JungfrauCtrl(run_dc, ctrl_src)\n",
+ "\n",
+ "if mem_cells < 0:\n",
+ " memory_cells, sc_start = ctrl_data.get_memory_cells()\n",
+ "\n",
+ " mem_cells_name = \"single cell\" if memory_cells == 1 else \"burst\"\n",
+ " print(f\"Run is in {mem_cells_name} mode.\\nStorage cell start: {sc_start:02d}\")\n",
+ "else:\n",
+ " memory_cells = mem_cells\n",
+ " mem_cells_name = \"single cell\" if memory_cells == 1 else \"burst\"\n",
+ " print(f\"Run is in manually set to {mem_cells_name} mode. With {memory_cells} memory cells\")\n",
+ "\n",
+ "if not manual_slow_data:\n",
+ " integration_time = ctrl_data.get_integration_time()\n",
+ " bias_voltage = ctrl_data.get_bias_voltage()\n",
+ " gain_setting = ctrl_data.get_gain_setting()\n",
+ " gain_mode = ctrl_data.get_gain_mode()\n",
+ "\n",
+ "print(f\"Integration time is {integration_time} us\")\n",
+ "print(f\"Gain setting is {gain_setting} (run settings: \"\n",
+ " f\"{ctrl_data.run_settings.value if ctrl_data.run_settings else ctrl_data.run_settings})\") # noqa\n",
+ "print(f\"Gain mode is {gain_mode}\")\n",
+ "print(f\"Bias voltage is {bias_voltage} V\")\n",
+ "print(f\"Number of memory cells are {memory_cells}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "condition = Conditions.Dark.jungfrau(\n",
+ " memory_cells=memory_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " integration_time=integration_time,\n",
+ " gain_setting=gain_setting,\n",
+ " gain_mode=gain_mode,\n",
+ ")\n",
+ "\n",
+ "def get_constants_for_module(mod: str):\n",
+ " \"\"\"Get calibration constants for given module for Jungfrau.\"\"\"\n",
+ " retrieval_function = partial(\n",
+ " get_from_db,\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=mod,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout,\n",
+ " verbosity=0,\n",
+ " meta_only=True,\n",
+ " load_data=False,\n",
+ " empty_constant=None\n",
+ " )\n",
+ "\n",
+ " mdata_dict = dict()\n",
+ " mdata_dict[\"constants\"] = dict()\n",
+ " constants = [\n",
+ " \"Offset\", \"BadPixelsDark\",\n",
+ " \"RelativeGain\", \"BadPixelsFF\",\n",
+ " ]\n",
+ " for cname in constants:\n",
+ " mdata_dict[\"constants\"][cname] = dict()\n",
+ " if not relative_gain and cname in [\"BadPixelsFF\", \"RelativeGain\"]:\n",
+ " continue\n",
+ " _, mdata = retrieval_function(\n",
+ " condition=condition,\n",
+ " constant=getattr(Constants.jungfrau, cname)(),\n",
+ " )\n",
+ " mdata_const = mdata.calibration_constant_version\n",
+ " const_mdata = mdata_dict[\"constants\"][cname]\n",
+ " # check if constant was successfully retrieved.\n",
+ " if mdata.comm_db_success:\n",
+ " const_mdata[\"file-path\"] = (\n",
+ " f\"{mdata_const.hdf5path}\" f\"{mdata_const.filename}\"\n",
+ " )\n",
+ " const_mdata[\"dataset-name\"] = mdata_const.h5path\n",
+ " const_mdata[\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " mdata_dict[\"physical-detector-unit\"] = mdata_const.device_name\n",
+ " else:\n",
+ " const_mdata[\"file-path\"] = None\n",
+ " const_mdata[\"creation-time\"] = None\n",
+ " return mdata_dict, mod"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n",
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
+ "# Avoid retrieving constants for available modules in calibration_metadata.yml\n",
+ "# This is used during reproducability.\n",
+ "query_karabo_da = []\n",
+ "for mod in karabo_da:\n",
+ " if mod in retrieved_constants.keys():\n",
+ " print(f\"Constant for {mod} already in \"\n",
+ " \"calibration_metadata.yml, won't query again.\")\n",
+ " continue\n",
+ " query_karabo_da.append(mod)\n",
+ "\n",
+ "with multiprocessing.Pool() as pool:\n",
+ " results = pool.map(get_constants_for_module, query_karabo_da)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "timestamps = dict()\n",
+ "for md_dict, mod in results:\n",
+ " retrieved_constants[mod] = md_dict\n",
+ "\n",
+ " module_timestamps = timestamps[mod] = dict()\n",
+ " module_constants = retrieved_constants[mod]\n",
+ "\n",
+ " print(f\"Module: {mod}:\")\n",
+ " for cname, mdata in module_constants[\"constants\"].items():\n",
+ " if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
+ " mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
+ " print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
+ "\n",
+ " for cname in [\"Offset\", \"BadPixelsDark\", \"RelativeGain\", \"BadPixelsFF\"]:\n",
+ " if cname in module_constants[\"constants\"]:\n",
+ " module_timestamps[cname] = module_constants[\"constants\"][cname][\"creation-time\"]\n",
+ " else:\n",
+ " module_timestamps[cname] = \"NA\"\n",
+ "\n",
+ "time_summary = retrieved_constants.setdefault(\"time-summary\", {})\n",
+ "time_summary = timestamps\n",
+ "\n",
+ "metadata.save()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.8.11 ('.cal4_venv')",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ },
+ "orig_nbformat": 4,
+ "vscode": {
+ "interpreter": {
+ "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/LPD/LPDChar_Darks_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
index 008620e36e2ea2c14a77897ffc2ea5554232c924..277d5231cc5b627733c378eeacefab5d952134ec 100644
--- a/notebooks/LPD/LPDChar_Darks_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
@@ -330,7 +330,7 @@
" offset_g[cap][qm] = np.zeros(\n",
" (offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
" noise_g[cap][qm] = np.zeros_like(offset_g[cap][qm])\n",
- " badpix_g[cap][qm] = np.zeros_like(offset_g[cap][qm])\n",
+ " badpix_g[cap][qm] = np.zeros_like(offset_g[cap][qm], dtype=np.uint32)\n",
" data_g[cap][qm] = np.full((ntrains, 3), np.nan)\n",
" ntest_g[cap][qm] = np.zeros_like(offset_g[cap][qm])\n",
"\n",
@@ -722,7 +722,8 @@
" title = '{} value'.format(const)\n",
" if const == 'BadPixelsDark':\n",
" vmax = 4\n",
- " data[data == 0] = np.nan\n",
+ " bpix_code = data.astype(np.float32)\n",
+ " bpix_code[bpix_code == 0] = np.nan\n",
" title = 'Bad pixel code'\n",
" label = title\n",
"\n",
@@ -731,12 +732,12 @@
" '3 {}'.format(BadPixels.OFFSET_OUT_OF_THRESHOLD.name),\n",
" '4 {}'.format('MIXED')]\n",
"\n",
- " heatmapPlot(data, add_panels=False, cmap='viridis',\n",
+ " heatmapPlot(bpix_code, add_panels=False, cmap='viridis',\n",
" y_label='Rows', x_label='Columns',\n",
" lut_label='', vmax=vmax,\n",
" use_axis=ax, cb_ticklabels=cb_labels, cb_ticks = np.arange(4)+1,\n",
" title='{}'.format(title))\n",
- "\n",
+ " del bpix_code\n",
" else:\n",
"\n",
" heatmapPlot(data, add_panels=False, cmap='viridis',\n",
@@ -1153,17 +1154,16 @@
" l_data_old = []\n",
" \n",
" data = np.copy(res[cap][qm]['BadPixelsDark'][:,:,:,gain])\n",
- " datau32 = data.astype(np.uint32)\n",
- " l_data.append(len(datau32[datau32>0].flatten()))\n",
+ " l_data.append(len(data[data>0].flatten()))\n",
" for bit in bits:\n",
- " l_data.append(np.count_nonzero(badpix_g[cap][qm][:,:,:,gain].astype(np.uint32) & bit.value))\n",
+ " l_data.append(np.count_nonzero(badpix_g[cap][qm][:,:,:,gain] & bit.value))\n",
" \n",
" if old_const[cap][qm]['BadPixelsDark'] is not None:\n",
+ " old_const[cap][qm]['BadPixelsDark'] = old_const[cap][qm]['BadPixelsDark'].astype(np.uint32)\n",
" dataold = np.copy(old_const[cap][qm]['BadPixelsDark'][:, :, :, gain])\n",
- " datau32old = dataold.astype(np.uint32)\n",
- " l_data_old.append(len(datau32old[datau32old>0].flatten()))\n",
+ " l_data_old.append(len(dataold[dataold>0].flatten()))\n",
" for bit in bits:\n",
- " l_data_old.append(np.count_nonzero(old_const[cap][qm]['BadPixelsDark'][:, :, :, gain].astype(np.uint32) & bit.value))\n",
+ " l_data_old.append(np.count_nonzero(old_const[cap][qm]['BadPixelsDark'][:, :, :, gain] & bit.value))\n",
"\n",
" l_data_name = ['All bad pixels', 'NOISE_OUT_OF_THRESHOLD', \n",
" 'OFFSET_OUT_OF_THRESHOLD', 'OFFSET_NOISE_EVAL_ERROR']\n",
@@ -1253,7 +1253,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.7"
+ "version": "3.8.12"
}
},
"nbformat": 4,
diff --git a/notebooks/LPD/LPD_Correct_Fast.ipynb b/notebooks/LPD/LPD_Correct_Fast.ipynb
index 6dc356abeaf13a79ad985ee885515f43e4682106..73906ff0388a22ba6d74b71bb474fc17d8003081 100644
--- a/notebooks/LPD/LPD_Correct_Fast.ipynb
+++ b/notebooks/LPD/LPD_Correct_Fast.ipynb
@@ -27,7 +27,7 @@
"sequences = [-1] # Sequences to correct, use [-1] for all\n",
"modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty\n",
"karabo_da = [''] # Data aggregators names to correct, use [''] for all\n",
- "run = 10 # runs to process, required\n",
+ "run = 10 # run to process, required\n",
"\n",
"# Source parameters\n",
"karabo_id = 'FXE_DET_LPD1M-1' # Karabo domain for detector.\n",
@@ -35,7 +35,7 @@
"output_source = '' # Output fast data source, empty to use same as input.\n",
"\n",
"# CalCat parameters\n",
- "use_dir_creation_date = True # Use the creation date of the directory for database time derivation.\n",
+ "creation_time = \"\" # The timestamp to use with Calibration DB. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
"cal_db_interface = '' # Not needed, compatibility with current webservice.\n",
"cal_db_timeout = 0 # Not needed, compatbility with current webservice.\n",
"cal_db_root = '/gpfs/exfel/d/cal/caldb_store'\n",
@@ -105,7 +105,11 @@
"from extra_data.components import LPD1M\n",
"\n",
"from cal_tools.lpdalgs import correct_lpd_frames\n",
- "from cal_tools.tools import CalibrationMetadata, get_dir_creation_date, write_compressed_frames\n",
+ "from cal_tools.tools import (\n",
+ " CalibrationMetadata,\n",
+ " calcat_creation_time,\n",
+ " write_compressed_frames,\n",
+ " )\n",
"from cal_tools.files import DataFile\n",
"from cal_tools.restful_config import restful_config"
]
@@ -135,12 +139,7 @@
"\n",
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"\n",
- "if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run) \n",
- "else:\n",
- " from datetime import datetime\n",
- " creation_time = datetime.now()\n",
- " \n",
+ "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
"print(f'Using {creation_time.isoformat()} as creation time')\n",
"\n",
"# Pick all modules/aggregators or those selected.\n",
@@ -224,63 +223,88 @@
"metadata": {},
"outputs": [],
"source": [
- "dark_calibrations = {\n",
- " 1: 'Offset', # np.float32\n",
- " 14: 'BadPixelsDark' # should be np.uint32, but is np.float64\n",
- "}\n",
- "\n",
- "dark_condition = [\n",
- " dict(parameter_id=1, value=bias_voltage), # Sensor bias voltage\n",
- " dict(parameter_id=7, value=mem_cells), # Memory cells\n",
- " dict(parameter_id=15, value=capacitor), # Feedback capacitor\n",
- " dict(parameter_id=13, value=256), # Pixels X\n",
- " dict(parameter_id=14, value=256), # Pixels Y\n",
- "]\n",
- "\n",
- "illuminated_calibrations = {\n",
- " 20: 'BadPixelsFF', # np.uint32\n",
- " 42: 'GainAmpMap', # np.float32\n",
- " 43: 'FFMap', # np.float32\n",
- " 44: 'RelativeGain' # np.float32\n",
- "}\n",
- "\n",
- "illuminated_condition = dark_condition.copy()\n",
- "illuminated_condition += [\n",
- " dict(parameter_id=3, value=photon_energy), # Source energy\n",
- " dict(parameter_id=25, value=category) # category\n",
- "]\n",
- "\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
+ "const_yaml = metadata.setdefault(\"retrieved-constants\", {})"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"const_data = {}\n",
"const_load_mp = psh.ProcessContext(num_workers=24)\n",
"\n",
- "print('Querying calibration database', end='', flush=True)\n",
- "start = perf_counter()\n",
- "for calibrations, condition in [\n",
- " (dark_calibrations, dark_condition),\n",
- " (illuminated_calibrations, illuminated_condition)\n",
- "]:\n",
- " resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n",
- " client, karabo_id, list(calibrations.keys()),\n",
- " {'parameters_conditions_attributes': condition},\n",
- " karabo_da='', event_at=creation_time.isoformat(), snapshot_at=None)\n",
- "\n",
- " if not resp['success']:\n",
- " raise RuntimeError(resp)\n",
- "\n",
- " for ccv in resp['data']:\n",
- " cc = ccv['calibration_constant']\n",
- " da = ccv['physical_detector_unit']['karabo_da']\n",
- " calibration_name = calibrations[cc['calibration_id']]\n",
- " \n",
- " dtype = np.uint32 if calibration_name.startswith('BadPixels') else np.float32\n",
- " \n",
- " const_data[(da, calibration_name)] = dict(\n",
- " path=Path(ccv['path_to_file']) / ccv['file_name'],\n",
- " dataset=ccv['data_set_name'],\n",
- " data=const_load_mp.alloc(shape=(256, 256, mem_cells, 3), dtype=dtype)\n",
- " )\n",
- " print('.', end='', flush=True)\n",
- " \n",
+ "if const_yaml: # Read constants from YAML file.\n",
+ " start = perf_counter()\n",
+ " for da, ccvs in const_yaml.items():\n",
+ "\n",
+ " for calibration_name, ccv in ccvs['constants'].items():\n",
+ "\n",
+ " dtype = np.uint32 if calibration_name.startswith('BadPixels') else np.float32\n",
+ "\n",
+ " const_data[(da, calibration_name)] = dict(\n",
+ " path=Path(ccv['file-path']),\n",
+ " dataset=ccv['dataset-name'],\n",
+ " data=const_load_mp.alloc(shape=(256, 256, mem_cells, 3), dtype=dtype)\n",
+ " )\n",
+ "else: # Retrieve constants from CALCAT.\n",
+ " dark_calibrations = {\n",
+ " 1: 'Offset', # np.float32\n",
+ " 14: 'BadPixelsDark' # should be np.uint32, but is np.float64\n",
+ " }\n",
+ "\n",
+ " dark_condition = [\n",
+ " dict(parameter_id=1, value=bias_voltage), # Sensor bias voltage\n",
+ " dict(parameter_id=7, value=mem_cells), # Memory cells\n",
+ " dict(parameter_id=15, value=capacitor), # Feedback capacitor\n",
+ " dict(parameter_id=13, value=256), # Pixels X\n",
+ " dict(parameter_id=14, value=256), # Pixels Y\n",
+ " ]\n",
+ "\n",
+ " illuminated_calibrations = {\n",
+ " 20: 'BadPixelsFF', # np.uint32\n",
+ " 42: 'GainAmpMap', # np.float32\n",
+ " 43: 'FFMap', # np.float32\n",
+ " 44: 'RelativeGain' # np.float32\n",
+ " }\n",
+ "\n",
+ " illuminated_condition = dark_condition.copy()\n",
+ " illuminated_condition += [\n",
+ " dict(parameter_id=3, value=photon_energy), # Source energy\n",
+ " dict(parameter_id=25, value=category) # category\n",
+ " ]\n",
+ "\n",
+ " print('Querying calibration database', end='', flush=True)\n",
+ " start = perf_counter()\n",
+ " for calibrations, condition in [\n",
+ " (dark_calibrations, dark_condition),\n",
+ " (illuminated_calibrations, illuminated_condition)\n",
+ " ]:\n",
+ " resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n",
+ " client, karabo_id, list(calibrations.keys()),\n",
+ " {'parameters_conditions_attributes': condition},\n",
+ " karabo_da='', event_at=creation_time.isoformat(), snapshot_at=None)\n",
+ "\n",
+ " if not resp['success']:\n",
+ " raise RuntimeError(resp)\n",
+ "\n",
+ " for ccv in resp['data']:\n",
+ " cc = ccv['calibration_constant']\n",
+ " da = ccv['physical_detector_unit']['karabo_da']\n",
+ " calibration_name = calibrations[cc['calibration_id']]\n",
+ " \n",
+ " dtype = np.uint32 if calibration_name.startswith('BadPixels') else np.float32\n",
+ " \n",
+ " const_data[(da, calibration_name)] = dict(\n",
+ " path=Path(ccv['path_to_file']) / ccv['file_name'],\n",
+ " dataset=ccv['data_set_name'],\n",
+ " data=const_load_mp.alloc(shape=(256, 256, mem_cells, 3), dtype=dtype)\n",
+ " )\n",
+ " print('.', end='', flush=True)\n",
+ " \n",
"total_time = perf_counter() - start\n",
"print(f'{total_time:.1f}s')"
]
diff --git a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..21ffbfb4d987be1746abed7145aed2f43850c64b
--- /dev/null
+++ b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
@@ -0,0 +1,221 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# LPD Retrieving Constants Pre-correction #\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "The following notebook provides a constants metadata in a YAML file to use while correcting LPD images."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Input parameters\n",
+ "in_folder = \"/gpfs/exfel/exp/FXE/202201/p003073/raw/\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/LPD_test\" # the folder to output to, required\n",
+ "metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n",
+ "modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty\n",
+ "karabo_da = [''] # Data aggregators names to correct, use [''] for all\n",
+ "run = 10 # run to process, required\n",
+ "\n",
+ "# Source parameters\n",
+ "karabo_id = 'FXE_DET_LPD1M-1' # Karabo domain for detector.\n",
+ "\n",
+ "# CalCat parameters\n",
+ "creation_time = \"\" # The timestamp to use with Calibration DB. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
+ "\n",
+ "# Operating conditions\n",
+ "mem_cells = 512 # Memory cells, LPD constants are always taken with 512 cells.\n",
+ "bias_voltage = 250.0 # Detector bias voltage.\n",
+ "capacitor = '5pF' # Capacitor setting: 5pF or 50pF\n",
+ "photon_energy = 9.2 # Photon energy in keV.\n",
+ "category = 0 # Whom to blame."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from pathlib import Path\n",
+ "from time import perf_counter\n",
+ "\n",
+ "from calibration_client import CalibrationClient\n",
+ "from calibration_client.modules import CalibrationConstantVersion\n",
+ "\n",
+ "from cal_tools.tools import (\n",
+ " CalibrationMetadata,\n",
+ " calcat_creation_time,\n",
+ " save_constant_metadata,\n",
+ ")\n",
+ "from cal_tools.restful_config import restful_config"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "out_folder = Path(out_folder)\n",
+ "out_folder.mkdir(exist_ok=True)\n",
+ "\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
+ "\n",
+ "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
+ "print(f'Using {creation_time.isoformat()} as creation time')\n",
+ "\n",
+ "# Pick all modules/aggregators or those selected.\n",
+ "if not karabo_da or karabo_da == ['']:\n",
+ " if not modules or modules == [-1]:\n",
+ " modules = list(range(16))\n",
+ "\n",
+ " karabo_da = [f'LPD{i:02d}' for i in modules] "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dark_calibrations = {\n",
+ " 1: 'Offset',\n",
+ " 14: 'BadPixelsDark',\n",
+ "}\n",
+ "\n",
+ "dark_condition = [\n",
+ " dict(parameter_id=1, value=bias_voltage), # Sensor bias voltage\n",
+ " dict(parameter_id=7, value=mem_cells), # Memory cells\n",
+ " dict(parameter_id=15, value=capacitor), # Feedback capacitor\n",
+ " dict(parameter_id=13, value=256), # Pixels X\n",
+ " dict(parameter_id=14, value=256), # Pixels Y\n",
+ "]\n",
+ "\n",
+ "illuminated_calibrations = {\n",
+ " 20: 'BadPixelsFF',\n",
+ " 42: 'GainAmpMap',\n",
+ " 43: 'FFMap',\n",
+ " 44: 'RelativeGain',\n",
+ "}\n",
+ "\n",
+ "illuminated_condition = dark_condition.copy()\n",
+ "illuminated_condition += [\n",
+ " dict(parameter_id=3, value=photon_energy), # Source energy\n",
+ " dict(parameter_id=25, value=category) # category\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "const_data = {}\n",
+ "\n",
+ "print('Querying calibration database', end='', flush=True)\n",
+ "start = perf_counter()\n",
+ "for k_da in karabo_da:\n",
+ " pdu = None\n",
+ " if k_da in retrieved_constants:\n",
+ " print(f\"Constant for {k_da} already in {metadata.filename}, won't query again.\") # noqa\n",
+ " continue\n",
+ " retrieved_constants[k_da] = dict()\n",
+ " const_mdata = retrieved_constants[k_da][\"constants\"] = dict()\n",
+ " for calibrations, condition in [\n",
+ " (dark_calibrations, dark_condition),\n",
+ " (illuminated_calibrations, illuminated_condition)\n",
+ " ]:\n",
+ " resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n",
+ " client, karabo_id, list(calibrations.keys()),\n",
+ " {'parameters_conditions_attributes': condition},\n",
+ " karabo_da=k_da, event_at=creation_time.isoformat(), snapshot_at=None)\n",
+ "\n",
+ " if not resp[\"success\"]:\n",
+ " print(f\"ERROR: Constants {list(calibrations.values())} \"\n",
+ " f\"were not retrieved, {resp['app_info']}\")\n",
+ " for cname in calibrations.values():\n",
+ " const_mdata[cname] = dict()\n",
+ " const_mdata[cname][\"file-path\"] = None\n",
+ " const_mdata[cname][\"dataset-name\"] = None\n",
+ " const_mdata[cname][\"creation-time\"] = None \n",
+ " continue\n",
+ "\n",
+ " for ccv in resp[\"data\"]:\n",
+ " cc = ccv['calibration_constant']\n",
+ " cname = calibrations[cc['calibration_id']]\n",
+ " const_mdata[cname] = dict()\n",
+ " const_mdata[cname][\"file-path\"] = str(Path(ccv['path_to_file']) / ccv['file_name'])\n",
+ " const_mdata[cname][\"dataset-name\"] = ccv['data_set_name']\n",
+ " const_mdata[cname][\"creation-time\"] = ccv['begin_at']\n",
+ " pdu = ccv['physical_detector_unit']['physical_name']\n",
+ "\n",
+ " print('.', end='', flush=True)\n",
+ " retrieved_constants[k_da][\"physical-detector-unit\"] = pdu\n",
+ "metadata.save()\n",
+ "\n",
+ "total_time = perf_counter() - start\n",
+ "print(f'{total_time:.1f}s')\n",
+ "print(f\"Stored retrieved constants in {metadata.filename}\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.8.11 ('.cal4_venv')",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ },
+ "orig_nbformat": 4,
+ "vscode": {
+ "interpreter": {
+ "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/ePix100/Correction_ePix100_NBC.ipynb b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
index 396a2ed69da41c82a2f502b6b4bea698d552d4f2..cad001b9cf26eb9760d4aa9882822d4a4f7bf056 100644
--- a/notebooks/ePix100/Correction_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
@@ -18,7 +18,8 @@
"outputs": [],
"source": [
"in_folder = \"/gpfs/exfel/exp/CALLAB/202031/p900113/raw\" # input folder, required\n",
- "out_folder = \"\" # output folder, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/epix_correct\" # output folder, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"sequences_per_node = 1 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel\n",
"run = 9988 # which run to read data from, required\n",
@@ -33,21 +34,20 @@
"\n",
"# Parameters affecting writing corrected data.\n",
"chunk_size_idim = 1 # H5 chunking size of output data\n",
- "overwrite = True # overwrite output folder\n",
"\n",
"# Only for testing\n",
"limit_images = 0 # ONLY FOR TESTING. process only first N images, 0 - process all.\n",
"\n",
"# Parameters for the calibration database.\n",
- "use_dir_creation_date = True # date constants injected before directory creation time\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8025\" # calibration DB interface to use\n",
"cal_db_timeout = 300000 # timeout on caldb requests\n",
+ "creation_time = \"\" # The timestamp to use with Calibration DBe. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
"\n",
"# Conditions for retrieving calibration constants.\n",
"bias_voltage = 200 # bias voltage\n",
"in_vacuum = False # detector operated in vacuum\n",
- "fix_temperature = 290. # fix temperature to this value\n",
- "temp_deviations = 5. # temperature deviation for the constant operating conditions\n",
+ "integration_time = -1 # Detector integration time, Default value -1 to use the value from the slow data.\n",
+ "fix_temperature = 290 # fixed temperature value in Kelvin, Default value -1 to use the value from files.\n",
"gain_photon_energy = 9.0 # Photon energy used for gain calibration\n",
"photon_energy = 0. # Photon energy to calibrate in number of photons, 0 for calibration in keV\n",
"\n",
@@ -77,25 +77,26 @@
"outputs": [],
"source": [
"import tabulate\n",
- "import traceback\n",
"import warnings\n",
"\n",
"import h5py\n",
"import pasha as psh\n",
- "import multiprocessing\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
- "from IPython.display import Latex, Markdown, display\n",
+ "from IPython.display import Latex, display\n",
"from extra_data import RunDirectory, H5File\n",
"from pathlib import Path\n",
"\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
- "from XFELDetAna.plotting.util import prettyPlotting\n",
"from cal_tools import h5_copy_except\n",
+ "from cal_tools.epix100 import epix100lib\n",
"from cal_tools.tools import (\n",
- " get_constant_from_db,\n",
+ " calcat_creation_time,\n",
" get_dir_creation_date,\n",
+ " get_constant_from_db,\n",
+ " load_specified_constants,\n",
+ " CalibrationMetadata,\n",
")\n",
"from cal_tools.step_timing import StepTimer\n",
"from iCalibrationDB import (\n",
@@ -116,12 +117,13 @@
"metadata": {},
"outputs": [],
"source": [
- "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
+ "x = 708 # rows of the ePix100\n",
+ "y = 768 # columns of the ePix100\n",
"\n",
"if absolute_gain:\n",
" relative_gain = True\n",
"\n",
- "plot_unit = 'ADU'\n"
+ "plot_unit = 'ADU'"
]
},
{
@@ -130,25 +132,34 @@
"metadata": {},
"outputs": [],
"source": [
- "x = 708 # rows of the ePix100\n",
- "y = 768 # columns of the ePix100\n",
- "\n",
"in_folder = Path(in_folder)\n",
- "ped_dir = in_folder / f\"r{run:04d}\"\n",
- "run_dc = RunDirectory(ped_dir, _use_voview=False)\n",
+ "out_folder = Path(out_folder)\n",
+ "\n",
+ "out_folder.mkdir(parents=True, exist_ok=True)\n",
+ "\n",
+ "run_folder = in_folder / f\"r{run:04d}\"\n",
"\n",
- "fp_name = path_template.format(run, karabo_da)\n",
+ "instrument_src = instrument_source_template.format(\n",
+ " karabo_id, receiver_template)\n",
"\n",
- "print(f\"Reading from: {ped_dir / fp_name}\")\n",
- "print(f\"Run is: {run}\")\n",
+ "print(f\"Correcting run: {run_folder}\")\n",
"print(f\"Instrument H5File source: {instrument_src}\")\n",
- "print(f\"Data corrected files are stored at: {out_folder}\")\n",
+ "print(f\"Data corrected files are stored at: {out_folder}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
+ "print(f\"Using {creation_time.isoformat()} as creation time\")\n",
"\n",
- "creation_time = None\n",
- "if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run)\n",
- "if creation_time:\n",
- " print(f\"Using {creation_time.isoformat()} as creation time\")"
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# Constant paths are saved under retrieved-constants in calibration_metadata.yml.\n",
+ "# NOTE: this notebook shouldn't overwrite calibration metadata file.\n",
+ "const_yaml = metadata.get(\"retrieved-constants\", {})"
]
},
{
@@ -157,6 +168,8 @@
"metadata": {},
"outputs": [],
"source": [
+ "run_dc = RunDirectory(run_folder, _use_voview=False)\n",
+ "\n",
"seq_files = [Path(f.filename) for f in run_dc.select(f\"*{karabo_id}*\").files]\n",
"\n",
"# If a set of sequences requested to correct,\n",
@@ -195,33 +208,31 @@
"run_parallel = False\n",
"\n",
"# Read control data.\n",
- "integration_time = int(run_dc.get_run_value(\n",
- " f\"{karabo_id}/DET/CONTROL\",\n",
- " \"expTime.value\"))\n",
- "temperature = np.mean(run_dc.get_array(\n",
- " f\"{karabo_id}/DET/{receiver_template}:daqOutput\",\n",
- " f\"data.backTemp\").values) / 100.\n",
- "\n",
- "if fix_temperature != 0:\n",
- " temperature_k = fix_temperature\n",
- " print(\"Temperature is fixed!\")\n",
+ "ctrl_data = epix100lib.epix100Ctrl(\n",
+ " run_dc=run_dc,\n",
+ " instrument_src=f\"{karabo_id}/DET/{receiver_template}:daqOutput\",\n",
+ " ctrl_src=f\"{karabo_id}/DET/CONTROL\",\n",
+ " )\n",
+ "\n",
+ "if integration_time < 0:\n",
+ " integration_time = ctrl_data.get_integration_time()\n",
+ " integration_time_str_add = \"\"\n",
"else:\n",
+ " integration_time_str_add = \"(manual input)\"\n",
+ "\n",
+ "if fix_temperature < 0:\n",
+ " temperature = ctrl_data.get_temprature()\n",
" temperature_k = temperature + 273.15\n",
+ " temp_str_add = \"\"\n",
+ "else:\n",
+ " temperature_k = fix_temperature\n",
+ " temperature = fix_temperature - 273.15\n",
+ " temp_str_add = \"(manual input)\"\n",
"\n",
"print(f\"Bias voltage is {bias_voltage} V\")\n",
- "print(f\"Detector integration time is set to {integration_time}\")\n",
- "print(\n",
- " f\"Mean temperature was {temperature:0.2f} °C \"\n",
- " f\"/ {temperature_k:0.2f} K at beginning of the run.\"\n",
- ")\n",
- "print(f\"Operated in vacuum: {in_vacuum} \")\n",
- "\n",
- "out_folder = Path(out_folder)\n",
- "if out_folder.is_dir() and not overwrite:\n",
- " raise AttributeError(\n",
- " \"Output path exists! Exiting as overwriting corrected data is prohibited.\")\n",
- "out_folder.mkdir(parents=True, exist_ok=True)\n",
- "step_timer.done_step(f'Reading control parameters.')\n"
+ "print(f\"Detector integration time is set to {integration_time} \\u03BCs {integration_time_str_add}\")\n",
+ "print(f\"Mean temperature: {temperature:0.2f}°C / {temperature_k:0.2f} K {temp_str_add}\")\n",
+ "print(f\"Operated in vacuum: {in_vacuum}\")"
]
},
{
@@ -276,38 +287,7 @@
" \"Offset\": dark_condition,\n",
" \"Noise\": dark_condition,\n",
" \"RelativeGain\": illum_condition,\n",
- "}\n",
- "\n",
- "const_data = dict()\n",
- "\n",
- "for cname, condition in const_cond.items():\n",
- " if cname == \"RelativeGain\" and not relative_gain:\n",
- " continue\n",
- " # TODO: Fix this logic.\n",
- " for parm in condition.parameters:\n",
- " if parm.name == \"Sensor Temperature\":\n",
- " parm.lower_deviation = temp_deviations\n",
- " parm.upper_deviation = temp_deviations\n",
- "\n",
- " const_data[cname] = get_constant_from_db(\n",
- " karabo_id=karabo_id,\n",
- " karabo_da=karabo_da,\n",
- " constant=getattr(Constants.ePix100, cname)(),\n",
- " condition=condition,\n",
- " empty_constant=None,\n",
- " cal_db_interface=cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " print_once=2,\n",
- " timeout=cal_db_timeout\n",
- " )\n",
- "\n",
- "if relative_gain and const_data[\"RelativeGain\"] is None:\n",
- " print(\n",
- " \"WARNING: RelativeGain map is requested, but not found.\\n\"\n",
- " \"No gain correction will be applied\"\n",
- " )\n",
- " relative_gain = False\n",
- " absolute_gain = False\n"
+ "}"
]
},
{
@@ -316,6 +296,43 @@
"metadata": {},
"outputs": [],
"source": [
+ "if const_yaml: # Used while reproducing corrected data.\n",
+ " print(f\"Using stored constants in {metadata.filename}\")\n",
+ " const_data, _ = load_specified_constants(const_yaml[karabo_da][\"constants\"])\n",
+ "else: # First correction attempt.\n",
+ " const_data = dict()\n",
+ " for cname, condition in const_cond.items():\n",
+ " # Avoid retrieving RelativeGain, if not needed for correction.\n",
+ " if cname == \"RelativeGain\" and not relative_gain:\n",
+ " const_data[cname] = None\n",
+ " else:\n",
+ " const_data[cname] = get_constant_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " constant=getattr(Constants.ePix100, cname)(),\n",
+ " condition=condition,\n",
+ " empty_constant=None,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " print_once=2,\n",
+ " timeout=cal_db_timeout\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if relative_gain and const_data.get(\"RelativeGain\", None) is None:\n",
+ " print(\n",
+ " \"WARNING: RelativeGain map is requested, but not found.\\n\"\n",
+ " \"No gain correction will be applied\"\n",
+ " )\n",
+ " relative_gain = False\n",
+ " absolute_gain = False\n",
+ "\n",
"# Initializing some parameters.\n",
"hscale = 1\n",
"stats = True\n",
diff --git a/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb b/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..591d3b84267c154a26b662327c34485b41172a74
--- /dev/null
+++ b/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
@@ -0,0 +1,229 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# ePix100 retrieve constants precorrection\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "The following notebook provides constants for the selected ePix100 modules before executing correction on the selected sequence files."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/CALLAB/202031/p900113/raw\" # input folder, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/epix_correct\" # output folder, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
+ "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
+ "run = 9988 # which run to read data from, required\n",
+ "\n",
+ "# Parameters for accessing the raw data.\n",
+ "karabo_id = \"MID_EXP_EPIX-1\" # Detector Karabo_ID\n",
+ "karabo_da = \"EPIX01\" # data aggregators\n",
+ "receiver_template = \"RECEIVER\" # detector receiver template for accessing raw data files\n",
+ "instrument_source_template = '{}/DET/{}:daqOutput' # instrument detector data source in h5files\n",
+ "\n",
+ "# Parameters for the calibration database.\n",
+ "creation_time = \"\" # The timestamp to use with Calibration DB. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
+ "cal_db_interface = \"tcp://max-exfl016:8015#8025\" # calibration DB interface to use\n",
+ "cal_db_timeout = 300000 # timeout on CalibrationDB requests\n",
+ "\n",
+ "# Conditions for retrieving calibration constants.\n",
+ "bias_voltage = 200 # bias voltage\n",
+ "in_vacuum = False # detector operated in vacuum\n",
+ "fix_temperature = 290 # fixed temperature value in Kelvin. Default value -1 to use the value from files.\n",
+ "integration_time = -1 # Detector integration time, Default value -1 to use the value from the slow data.\n",
+ "gain_photon_energy = 9.0 # Photon energy used for gain calibration\n",
+ "\n",
+ "# Flags to select type of applied corrections.\n",
+ "relative_gain = True # Apply relative gain correction."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "from extra_data import RunDirectory\n",
+ "from pathlib import Path\n",
+ "\n",
+ "from cal_tools.epix100 import epix100lib\n",
+ "from cal_tools.tools import (\n",
+ " calcat_creation_time,\n",
+ " get_dir_creation_date,\n",
+ " get_from_db,\n",
+ " save_constant_metadata,\n",
+ " CalibrationMetadata,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = Path(in_folder)\n",
+ "out_folder = Path(out_folder)\n",
+ "\n",
+ "out_folder.mkdir(parents=True, exist_ok=True)\n",
+ "\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# NOTE: this notebook will not overwrite calibration metadata file,\n",
+ "# if it already contains details about which constants to use.\n",
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
+ "\n",
+ "if karabo_da in retrieved_constants:\n",
+ " print(\n",
+ " f\"Constant for {karabo_da} already in {metadata.filename}, won't query again.\"\n",
+ " ) \n",
+ " import sys\n",
+ "\n",
+ " sys.exit(0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
+ "print(f\"Using {creation_time.isoformat()} as creation time\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Read control data.\n",
+ "run_dc = RunDirectory(in_folder / f\"r{run:04d}\")\n",
+ "\n",
+ "ctrl_data = epix100lib.epix100Ctrl(\n",
+ " run_dc=run_dc,\n",
+ " instrument_src=f\"{karabo_id}/DET/{receiver_template}:daqOutput\",\n",
+ " ctrl_src=f\"{karabo_id}/DET/CONTROL\",\n",
+ " )\n",
+ "\n",
+ "if integration_time < 0:\n",
+ " integration_time = ctrl_data.get_integration_time()\n",
+ " integration_time_str_add = \"\"\n",
+ "else:\n",
+ " integration_time_str_add = \"(manual input)\"\n",
+ "\n",
+ "if fix_temperature < 0:\n",
+ " temperature = ctrl_data.get_temprature()\n",
+ " temperature_k = temperature + 273.15\n",
+ " temp_str_add = \"\"\n",
+ "else:\n",
+ " temperature_k = fix_temperature\n",
+ " temperature = fix_temperature - 273.15\n",
+ " temp_str_add = \"(manual input)\"\n",
+ "\n",
+ "\n",
+ "print(f\"Bias voltage is {bias_voltage} V\")\n",
+ "print(f\"Detector integration time is set to {integration_time} \\u03BCs {integration_time_str_add}\")\n",
+ "print(f\"Mean temperature: {temperature:0.2f}°C / {temperature_k:0.2f} K {temp_str_add}\")\n",
+ "print(f\"Operated in vacuum: {in_vacuum}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cond_dict = {\n",
+ " \"bias_voltage\": bias_voltage,\n",
+ " \"integration_time\": integration_time,\n",
+ " \"temperature\": temperature_k,\n",
+ " \"in_vacuum\": in_vacuum,\n",
+ "}\n",
+ "\n",
+ "dark_condition = Conditions.Dark.ePix100(**cond_dict)\n",
+ "\n",
+ "# update conditions with illuminated conditions.\n",
+ "cond_dict.update({\"photon_energy\": gain_photon_energy})\n",
+ "\n",
+ "illum_condition = Conditions.Illuminated.ePix100(**cond_dict)\n",
+ "\n",
+ "const_cond = {\n",
+ " \"Offset\": dark_condition,\n",
+ " \"Noise\": dark_condition,\n",
+ " \"RelativeGain\": illum_condition,\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "const_data = dict()\n",
+ "mdata_dict = dict()\n",
+ "mdata_dict[\"constants\"] = dict()\n",
+ "for cname, condition in const_cond.items():\n",
+ " # Avoid retrieving RelativeGain, if not needed for correction.\n",
+ " if cname == \"RelativeGain\" and not relative_gain:\n",
+ " const_data[cname] = None\n",
+ " else:\n",
+ " const_data[cname], mdata = get_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " constant=getattr(Constants.ePix100, cname)(),\n",
+ " condition=condition,\n",
+ " empty_constant=None,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " verbosity=2,\n",
+ " timeout=cal_db_timeout,\n",
+ " meta_only=True,\n",
+ " )\n",
+ " save_constant_metadata(mdata_dict[\"constants\"], mdata, cname)\n",
+ "mdata_dict[\"physical-detector-unit\"] = mdata.calibration_constant_version.device_name\n",
+ "retrieved_constants[karabo_da] = mdata_dict\n",
+ "metadata.save()\n",
+ "print(f\"Stored retrieved constants in {metadata.filename}\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.8.11 ('.cal4_venv')",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ },
+ "orig_nbformat": 4,
+ "vscode": {
+ "interpreter": {
+ "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
index 3583a7f377e8ed64e56cb15cf7dcd39820940312..67b052ab358ec5f3fbb678fa062982c932ed09ba 100644
--- a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
+++ b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
@@ -14,7 +14,14 @@
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"karabo_id = \"SPB_DET_AGIPD1M-1\" # detector instance\n",
"gain_names = ['High gain', 'Medium gain', 'Low gain'] # a list of gain names to be used in plotting\n",
- "threshold_names = ['HG-MG threshold', 'MG_LG threshold'] # a list of gain names to be used in plotting"
+ "threshold_names = ['HG-MG threshold', 'MG_LG threshold'] # a list of gain names to be used in plotting\n",
+ "local_output = True # Boolean indicating that local constants were stored in the out_folder\n",
+ "\n",
+ "# Skip the whole notebook if local_output is false in the preceding notebooks.\n",
+ "if not local_output:\n",
+ " print('No local constants saved. Skipping summary plots')\n",
+ " import sys\n",
+ " sys.exit(0)"
]
},
{
@@ -52,15 +59,6 @@
"from XFELDetAna.plotting.simpleplot import simplePlot"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Note: this notebook assumes that local_output was set to True in the preceding characterization notebook"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -77,6 +75,9 @@
" elif \"HED\" in karabo_id:\n",
" dinstance = \"AGIPD500K\"\n",
" nmods = 8\n",
+ " # This list needs to be in that order as later Adaptive or fixed gain is\n",
+ " # decided based on the condition for the Offset constant.\n",
+ " expected_constants = ['Offset', 'Noise', 'ThresholdsDark', 'BadPixelsDark']\n",
" display(Markdown(\"\"\"\n",
" \n",
"# Summary of AGIPD dark characterization #\n",
@@ -130,6 +131,7 @@
"elif \"LPD\" in karabo_id:\n",
" dinstance = \"LPD1M1\"\n",
" nmods = 16\n",
+ " expected_constants = ['Offset', 'Noise', 'BadPixelsDark']\n",
" display(Markdown(\"\"\"\n",
" \n",
"# Summary of LPD dark characterization #\n",
@@ -173,6 +175,7 @@
"elif \"DSSC\" in karabo_id:\n",
" dinstance = \"DSSC1M1\"\n",
" nmods = 16\n",
+ " expected_constants = ['Offset', 'Noise', 'BadPixelsDark']\n",
" display(Markdown(\"\"\"\n",
" \n",
"# Summary of DSSC dark characterization #\n",
@@ -220,17 +223,23 @@
"old_cons = OrderedDict()\n",
"mod_names = []\n",
"\n",
+ "gain_mode = None if \"AGIPD\" in karabo_id else False\n",
+ "\n",
"# Loop over modules\n",
"for i in range(nmods):\n",
" qm = module_index_to_qm(i)\n",
" if not mod_mapping.get(qm):\n",
" continue\n",
" mod_pdu = mod_mapping[qm]\n",
- " # loop over constants\n",
- " for const in ['Offset', 'Noise', 'ThresholdsDark', 'BadPixelsDark']:\n",
+ " # Loop over expected dark constants in out_folder.\n",
+ " # Some constants can be missing in out_folder.\n",
+ " # e.g. ThresholdsDark for fixed gain AGIPD, DSSC and LPD.\n",
+ " for const in expected_constants:\n",
" # first load new constant\n",
" fpath = out_folder / f\"const_{const}_{mod_pdu}.h5\"\n",
- " \n",
+ " # No ThresholdsDark expected for AGIPD in fixed gain mode.\n",
+ " if const == \"ThresholdsDark\" and gain_mode:\n",
+ " continue\n",
" if not fpath.exists():\n",
" print(f\"No local output file {fpath} found\")\n",
" continue\n",
@@ -238,10 +247,13 @@
" with h5py.File(fpath, 'r') as f:\n",
" if qm not in data:\n",
" mod_names.append(qm)\n",
- " data[qm] = OrderedDict()\n",
- "\n",
- " data[qm][const] = f[\"data\"][()]\n",
+ " data.setdefault(qm, OrderedDict())[const] = f[\"data\"][()]\n",
"\n",
+ " if gain_mode is None:\n",
+ " if 'Gain Mode' in f['condition'].keys():\n",
+ " gain_mode = bool(f[\"condition\"][\"Gain Mode\"][\"value\"][()])\n",
+ " else:\n",
+ " gain_mode = False\n",
" # try finding old constants using paths from CalCat store\n",
" qm_mdata = old_constant_metadata[qm]\n",
"\n",
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index c1350f12e8e562a1d0c2bdd143f39445112d910b..b479ac76af04500fc5566b2d2660601ca1f8f171 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -18,7 +18,8 @@
"outputs": [],
"source": [
"in_folder = \"/gpfs/exfel/exp/SQS/202031/p900166/raw\" # input folder\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove\" # output folder\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/pnccd_correct\" # output folder\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"run = 347 # which run to read data from\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"sequences_per_node = 1 # number of sequences running on the same slurm node.\n",
@@ -44,10 +45,10 @@
"integration_time = 70 # detector's integration time\n",
"photon_energy = 1.6 # Al fluorescence in keV\n",
"\n",
+ "# Parameters for the calibration database.\n",
"cal_db_interface = \"tcp://max-exfl016:8015\" # calibration DB interface to use\n",
"cal_db_timeout = 300000 # timeout on caldb requests\n",
- "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.ms e.g. 2019-07-04 11:02:41.00\n",
- "use_dir_creation_date = True # To obtain creation time of the run\n",
+ "creation_time = \"\" # The timestamp to use with Calibration DB. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
"\n",
"# Booleans for selecting corrections to apply.\n",
"only_offset = False # Only, apply offset.\n",
@@ -57,13 +58,9 @@
"\n",
"# parameters affecting stored output data.\n",
"chunk_size_idim = 1 # H5 chunking size of output data\n",
- "overwrite = True # keep this as True to not overwrite the output \n",
"# ONLY FOR TESTING\n",
"limit_images = 0 # this parameter is used for limiting number of images to correct from a sequence file. ONLY FOR TESTING.\n",
"\n",
- "# TODO: REMOVE\n",
- "db_module = \"\"\n",
- "\n",
"\n",
"def balance_sequences(in_folder, run, sequences, sequences_per_node, karabo_da):\n",
" from xfel_calibrate.calibrate import balance_sequences as bs\n",
@@ -76,12 +73,12 @@
"metadata": {},
"outputs": [],
"source": [
- "# Here the herarichy and dependability for correction booleans are defined \n",
+ "# Here the herarichy and dependability for correction booleans are defined\n",
"corr_bools = {}\n",
"\n",
"corr_bools[\"only_offset\"] = only_offset\n",
"\n",
- "# Dont apply any corrections if only_offset is requested.\n",
+ "# Apply offset only.\n",
"if not only_offset:\n",
" corr_bools[\"relgain\"] = relgain\n",
" corr_bools[\"common_mode\"] = common_mode\n",
@@ -98,7 +95,6 @@
"import os\n",
"import warnings\n",
"from pathlib import Path\n",
- "from typing import Tuple\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import h5py\n",
@@ -115,14 +111,16 @@
"from XFELDetAna import xfelpycaltools as xcal\n",
"from cal_tools import pnccdlib\n",
"from cal_tools.tools import (\n",
- " get_constant_from_db_and_time,\n",
+ " calcat_creation_time,\n",
" get_dir_creation_date,\n",
+ " get_constant_from_db_and_time,\n",
" get_random_db_interface,\n",
- " map_modules_from_folder,\n",
+ " load_specified_constants,\n",
+ " CalibrationMetadata,\n",
")\n",
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools import h5_copy_except\n",
- "from iCalibrationDB import Conditions, ConstantMetaData, Constants\n",
+ "from iCalibrationDB import Conditions, Constants\n",
"from iCalibrationDB.detectors import DetectorTypes"
]
},
@@ -138,6 +136,7 @@
"# Sensor size and block size definitions (important for common mode and other corrections):\n",
"pixels_x = 1024 # rows of pnCCD in pixels \n",
"pixels_y = 1024 # columns of pnCCD in pixels\n",
+ "in_folder = Path(in_folder)\n",
"sensorSize = [pixels_x, pixels_y]\n",
"# For xcal.HistogramCalculators.\n",
"blockSize = [pixels_x//2, pixels_y//2] # sensor area will be analysed according to blockSize.\n",
@@ -150,20 +149,7 @@
"print(f\"Instrument H5File source: {instrument_src}\\n\")\n",
"\n",
"# Run's creation time:\n",
- "if creation_time:\n",
- " try:\n",
- " creation_time = datetime.datetime.strptime(creation_time, '%Y-%m-%d %H:%M:%S.%f')\n",
- " except Exception as e:\n",
- " print(f\"creation_time value error: {e}.\" \n",
- " \"Use same format as YYYY-MM-DD HR:MN:SC.ms e.g. 2019-07-04 11:02:41.00/n\")\n",
- " creation_time = None\n",
- " print(\"Given creation time will not be used.\")\n",
- "else:\n",
- " creation_time = None\n",
- "\n",
- "if not creation_time and use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run)\n",
- "\n",
+ "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
"print(f\"Creation time: {creation_time}\")"
]
},
@@ -182,12 +168,20 @@
"metadata": {},
"outputs": [],
"source": [
- "run_dc = RunDirectory(Path(in_folder) / f\"r{run:04d}\", _use_voview=False)\n",
- "ctrl_data = pnccdlib.PnccdCtrl(run_dc, karabo_id)\n",
+ "run_dc = RunDirectory(in_folder / f\"r{run:04d}\", _use_voview=False)\n",
+ "\n",
+ "# Output Folder Creation:\n",
+ "os.makedirs(out_folder, exist_ok=True)\n",
+ "\n",
+ "# NOTE: this notebook shouldn't overwrite calibration metadata file.\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n",
+ "const_yaml = metadata.get(\"retrieved-constants\", {})\n",
"\n",
"# extract control data\n",
"step_timer.start()\n",
"\n",
+ "ctrl_data = pnccdlib.PnccdCtrl(run_dc, karabo_id)\n",
"if bias_voltage == 0.:\n",
" bias_voltage = ctrl_data.get_bias_voltage()\n",
"if gain == -1:\n",
@@ -307,16 +301,6 @@
"b_range = Event_Bin_Dict[\"b_range\"]"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Output Folder Creation:\n",
- "os.makedirs(out_folder, exist_ok=True if overwrite else False)"
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -330,57 +314,52 @@
"metadata": {},
"outputs": [],
"source": [
- "def get_dark(db_parms: Tuple[str, int], bias_voltage: float, gain: float, integration_time: float,\n",
- " fix_temperature_top: float, creation_time: str, run: str) -> dict :\n",
- "# This function is to retrieve the dark constants associated with the run of interest:\n",
+ "display(Markdown(\"### Constants retrieval\"))\n",
+ "step_timer.start()\n",
"\n",
- " cal_db_interface, cal_db_timeout = db_parms\n",
- " cal_db_interface = get_random_db_interface(cal_db_interface)\n",
- " print(f'Calibration database interface: {cal_db_interface}')\n",
- " \n",
- " try: \n",
- " Offset = None\n",
- " Noise = None\n",
- " BadPixelsDark = None\n",
- "\n",
- " constants = {} \n",
- " constants['Offset'] = Offset\n",
- " constants['Noise'] = Noise\n",
- " constants['BadPixelsDark'] = BadPixelsDark\n",
- " when = {}\n",
- " \n",
- " condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " gain_setting=gain,\n",
- " temperature=fix_temperature_top,\n",
- " pixels_x=pixels_x,\n",
- " pixels_y=pixels_y)\n",
- " \n",
- " for const in constants.keys():\n",
- " constants[const], when[const] = \\\n",
- " get_constant_from_db_and_time(karabo_id, karabo_da,\n",
- " getattr(Constants.CCD(DetectorTypes.pnCCD), const)(),\n",
- " condition,\n",
- " np.zeros((pixels_x, pixels_y, 1)),\n",
- " cal_db_interface,\n",
- " creation_time=creation_time)\n",
- " for parm in condition.parameters:\n",
- " if parm.name == \"Sensor Temperature\":\n",
- " print(f\"For run {run}: dark images for offset calculation \" \n",
- " f\"were taken at temperature: {parm.value:0.2f} \"\n",
- " f\"K @ {when['Offset']}\")\n",
- " except Exception as e:\n",
- " print(f\"Error: {e}\")\n",
- "\n",
- "\n",
- " return constants"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Retrieving calibration constants"
+ "conditions_dict = {\n",
+ " \"bias_voltage\": bias_voltage,\n",
+ " \"integration_time\": integration_time,\n",
+ " \"gain_setting\": gain,\n",
+ " \"temperature\": fix_temperature_top,\n",
+ " \"pixels_x\": pixels_x,\n",
+ " \"pixels_y\": pixels_y,\n",
+ "}\n",
+ "# Dark condition\n",
+ "dark_condition = Conditions.Dark.CCD(**conditions_dict)\n",
+ "# Add photon energy.\n",
+ "conditions_dict.update({\"photon_energy\": photon_energy})\n",
+ "illum_condition = Conditions.Illuminated.CCD(**conditions_dict)\n",
+ "\n",
+ "# A dictionary for initializing constants. {cname: empty constant array}\n",
+ "empty_constants = {\n",
+ " \"Offset\": np.zeros((pixels_x, pixels_y, 1), dtype=np.float32),\n",
+ " \"Noise\": np.zeros((pixels_x, pixels_y, 1), dtype=np.float32),\n",
+ " \"BadPixelsDark\": np.zeros((pixels_x, pixels_y, 1), dtype=np.uint32),\n",
+ " \"RelativeGain\": np.zeros((pixels_x, pixels_y), dtype=np.float32),\n",
+ "}\n",
+ "\n",
+ "if const_yaml: # Used while reproducing corrected data.\n",
+ " print(f\"Using stored constants in {metadata.filename}\")\n",
+ " constants, when = load_specified_constants(\n",
+ " const_yaml[karabo_da][\"constants\"], empty_constants\n",
+ " )\n",
+ "else:\n",
+ " constants = dict()\n",
+ " when = dict()\n",
+ " for cname, cempty in empty_constants.items():\n",
+ " # No need for retrieving RelativeGain, if not used for correction.\n",
+ " if not corr_bools.get(\"relgain\") and cname == \"RelativeGain\":\n",
+ " continue\n",
+ " constants[cname], when[cname] = get_constant_from_db_and_time(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " constant=getattr(Constants.CCD(DetectorTypes.pnCCD), cname)(),\n",
+ " condition=illum_condition if cname == \"RelativeGain\" else dark_condition,\n",
+ " empty_constant=cempty,\n",
+ " cal_db_interface=get_random_db_interface(cal_db_interface),\n",
+ " creation_time=creation_time,\n",
+ " )"
]
},
{
@@ -389,13 +368,6 @@
"metadata": {},
"outputs": [],
"source": [
- "display(Markdown('### Dark constants retrieval'))\n",
- "step_timer.start()\n",
- "db_parms = cal_db_interface, cal_db_timeout\n",
- "\n",
- "constants = get_dark(db_parms, bias_voltage, gain, integration_time,\n",
- " fix_temperature_top, creation_time, run)\n",
- "\n",
"fig = xana.heatmapPlot(constants[\"Offset\"][:,:,0], x_label='Columns', y_label='Rows', lut_label='Offset (ADU)', \n",
" aspect=1, \n",
" x_range=(0, pixels_y), y_range=(0, pixels_x), vmax=16000, \n",
@@ -413,49 +385,16 @@
" aspect=1, x_range=(0, pixels_y), y_range=(0, pixels_x), \n",
" panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', \n",
" title = 'Dark Bad Pixels Map')\n",
- "step_timer.done_step(\"Dark constants retrieval\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
+ "\n",
"if corr_bools.get('relgain'):\n",
- " step_timer.start()\n",
- " display(Markdown('### Relative gain constant retrieval'))\n",
- " metadata = ConstantMetaData()\n",
- " relgain = Constants.CCD(DetectorTypes.pnCCD).RelativeGain()\n",
- " metadata.calibration_constant = relgain\n",
- " # set the operating condition\n",
- " condition = Conditions.Illuminated.CCD(bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " gain_setting=gain,\n",
- " temperature=fix_temperature_top,\n",
- " pixels_x=pixels_x,\n",
- " pixels_y=pixels_y, \n",
- " photon_energy=photon_energy)\n",
- "\n",
- " constants[\"RelativeGain\"], relgain_time = \\\n",
- " get_constant_from_db_and_time(karabo_id, karabo_da,\n",
- " Constants.CCD(DetectorTypes.pnCCD).RelativeGain(),\n",
- " condition,\n",
- " np.zeros((pixels_x, pixels_y)),\n",
- " cal_db_interface,\n",
- " creation_time=creation_time)\n",
- "\n",
- " print(f\"Retrieved RelativeGain constant with creation time: {relgain_time}\")\n",
- "\n",
- " display(Markdown('### Relative Gain Map Retrieval'))\n",
" fig = xana.heatmapPlot(constants[\"RelativeGain\"], figsize=(8, 8), x_label='Columns', y_label='Rows', \n",
- " lut_label='Relative Gain', \n",
- " aspect=1, x_range=(0, pixels_y), y_range=(0, pixels_x), vmin=0.8, vmax=1.2, \n",
- " panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', \n",
- " panel_top_low_lim = 0.5, panel_top_high_lim = 1.5, panel_side_low_lim = 0.5, \n",
- " panel_side_high_lim = 1.5, \n",
- " title = f'Relative Gain Map for pnCCD (Gain = 1/{int(gain)})')\n",
- " step_timer.done_step(\"Relative gain constant retrieval\")"
+ " lut_label='Relative Gain', \n",
+ " aspect=1, x_range=(0, pixels_y), y_range=(0, pixels_x), vmin=0.8, vmax=1.2, \n",
+ " panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', \n",
+ " panel_top_low_lim = 0.5, panel_top_high_lim = 1.5, panel_side_low_lim = 0.5, \n",
+ " panel_side_high_lim = 1.5, \n",
+ " title = f'Relative Gain Map for pnCCD (Gain = 1/{int(gain)})')\n",
+ "step_timer.done_step(\"Constants retrieval\")"
]
},
{
diff --git a/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb b/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..eeac52fe3e033eb3bfef5a82fc3376ca1e8aecf4
--- /dev/null
+++ b/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
@@ -0,0 +1,223 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# pnCCD retrieve constants precorrection\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "The following notebook provides constants for the selected pnCCD modules before executing correction on the selected sequence files."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/SQS/202031/p900166/raw\" # input folder\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/pnccd_correct\" # output folder\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
+ "run = 347 # which run to read data from\n",
+ "sequences = [0] # sequences to correct, set to -1 for all, range allowed\n",
+ "\n",
+ "karabo_da = 'PNCCD01' # data aggregators\n",
+ "karabo_id = \"SQS_NQS_PNCCD1MP\" # detector Karabo_ID\n",
+ "\n",
+ "# Conditions for retrieving calibration constants\n",
+ "fix_temperature_top = 0. # fix temperature for top sensor in K, set to 0. to use value from slow data.\n",
+ "fix_temperature_bot = 0. # fix temperature for bottom sensor in K, set to 0. to use value from slow data.\n",
+ "gain = -1 # the detector's gain setting. Set to -1 to use the value from the slow data.\n",
+ "bias_voltage = 0. # the detector's bias voltage. set to 0. to use value from slow data.\n",
+ "integration_time = 70 # detector's integration time\n",
+ "photon_energy = 1.6 # Al fluorescence in keV\n",
+ "\n",
+ "# Parameters for the calibration database.\n",
+ "cal_db_interface = \"tcp://max-exfl016:8015\" # calibration DB interface to use\n",
+ "cal_db_timeout = 300000 # timeout on CalibrationDB requests\n",
+ "creation_time = \"\" # The timestamp to use with Calibration DBe. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
+ "\n",
+ "# Booleans for selecting corrections to apply.\n",
+ "only_offset = False # Only, apply offset.\n",
+ "relgain = True # Apply relative gain correction\n",
+ "\n",
+ "# parameters affecting stored output data.\n",
+ "overwrite = True # keep this as True to not overwrite the output "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "from pathlib import Path\n",
+ "\n",
+ "from IPython.display import Markdown, display\n",
+ "from extra_data import RunDirectory\n",
+ "\n",
+ "from cal_tools import pnccdlib\n",
+ "from cal_tools.tools import (\n",
+ " calcat_creation_time,\n",
+ " get_dir_creation_date,\n",
+ " get_from_db,\n",
+ " get_random_db_interface,\n",
+ " save_constant_metadata,\n",
+ " CalibrationMetadata,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants\n",
+ "from iCalibrationDB.detectors import DetectorTypes"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# NOTE: this notebook will not overwrite calibration metadata file,\n",
+ "# if it already contains details about which constants to use.\n",
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
+ "if karabo_da in retrieved_constants:\n",
+ " print(\n",
+ " f\"Constant for {karabo_da} already in {metadata.filename}, won't query again.\"\n",
+ " ) # noqa\n",
+ " import sys\n",
+ "\n",
+ " sys.exit(0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Here the correction booleans dictionary is defined\n",
+ "corr_bools = {}\n",
+ "\n",
+ "corr_bools[\"only_offset\"] = only_offset\n",
+ "\n",
+ "# Apply offset only.\n",
+ "if not only_offset:\n",
+ " corr_bools[\"relgain\"] = relgain"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"Calibration database interface selected: {cal_db_interface}\")\n",
+ "\n",
+ "# Run's creation time:\n",
+ "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
+ "print(f\"Creation time: {creation_time}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_dc = RunDirectory(Path(in_folder) / f\"r{run:04d}\", _use_voview=False)\n",
+ "ctrl_data = pnccdlib.PnccdCtrl(run_dc, karabo_id)\n",
+ "\n",
+ "# extract control data\n",
+ "if bias_voltage == 0.0:\n",
+ " bias_voltage = ctrl_data.get_bias_voltage()\n",
+ "if gain == -1:\n",
+ " gain = ctrl_data.get_gain()\n",
+ "if fix_temperature_top == 0:\n",
+ " fix_temperature_top = ctrl_data.get_fix_temperature_top()\n",
+ "\n",
+ "# Printing the Parameters Read from the Data File:\n",
+ "display(Markdown(\"### Detector Parameters\"))\n",
+ "print(f\"Bias voltage: {bias_voltage:0.1f} V.\")\n",
+ "print(f\"Detector gain: {int(gain)}.\")\n",
+ "print(f\"Detector integration time: {integration_time} ms\")\n",
+ "print(f\"Top pnCCD sensor temperature: {fix_temperature_top:0.2f} K\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown(\"### Constants retrieval\"))\n",
+ "\n",
+ "conditions_dict = {\n",
+ " \"bias_voltage\": bias_voltage,\n",
+ " \"integration_time\": integration_time,\n",
+ " \"gain_setting\": gain,\n",
+ " \"temperature\": fix_temperature_top,\n",
+ " \"pixels_x\": 1024,\n",
+ " \"pixels_y\": 1024,\n",
+ "}\n",
+ "# Dark condition\n",
+ "dark_condition = Conditions.Dark.CCD(**conditions_dict)\n",
+ "# Add photon energy.\n",
+ "conditions_dict.update({\"photon_energy\": photon_energy})\n",
+ "illum_condition = Conditions.Illuminated.CCD(**conditions_dict)\n",
+ "\n",
+ "mdata_dict = dict()\n",
+ "mdata_dict[\"constants\"] = dict()\n",
+ "for cname in [\"Offset\", \"Noise\", \"BadPixelsDark\", \"RelativeGain\"]:\n",
+ " # No need for retrieving RelativeGain, if not used for correction.\n",
+ " if not corr_bools.get(\"relgain\") and cname == \"RelativeGain\":\n",
+ " continue\n",
+ " _, mdata = get_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " constant=getattr(Constants.CCD(DetectorTypes.pnCCD), cname)(),\n",
+ " condition=illum_condition if cname == \"RelativeGain\" else dark_condition,\n",
+ " empty_constant=None,\n",
+ " cal_db_interface=get_random_db_interface(cal_db_interface),\n",
+ " creation_time=creation_time,\n",
+ " verbosity=1,\n",
+ " load_data=False,\n",
+ " )\n",
+ " save_constant_metadata(mdata_dict[\"constants\"], mdata, cname)\n",
+ "\n",
+ "mdata_dict[\"physical-detector-unit\"] = mdata.calibration_constant_version.device_name\n",
+ "retrieved_constants[karabo_da] = mdata_dict\n",
+ "metadata.save()\n",
+ "print(f\"Stored retrieved constants in {metadata.filename}\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.8.11 ('.cal4_venv')",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ },
+ "orig_nbformat": 4,
+ "vscode": {
+ "interpreter": {
+ "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/setup.py b/setup.py
index 9f91ce5d93e84c29e7bc39beaa83b84c78b1e53d..aae7823b411bdbf59ab410e4e3b2d4d4fe510821 100644
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,6 @@ from subprocess import check_output
import numpy
from Cython.Build import cythonize
-from Cython.Distutils import build_ext
from setuptools import find_packages, setup
from setuptools.extension import Extension
@@ -24,7 +23,12 @@ ext_modules = [
extra_compile_args=['-O3', '-fopenmp', '-march=native',
'-ftree-vectorize', '-frename-registers'],
extra_link_args=['-fopenmp'],
- )
+ ),
+ Extension(
+ "cal_tools.gotthard2.gotthard2algs",
+ ["src/cal_tools/gotthard2/gotthard2algs.pyx"],
+ include_dirs=[numpy.get_include()],
+ ),
]
@@ -55,10 +59,12 @@ install_requires = [
"astcheck==0.2.5",
"astsearch==0.2.0",
"cfelpyutils==1.0.1",
+ "calibration_client==10.0.0",
"dill==0.3.0",
"docutils==0.17.1",
"dynaconf==3.1.4",
- "extra_data==1.11.0",
+ "env_cache==0.1",
+ "extra_data==1.12.0",
"extra_geom==1.6.0",
"gitpython==3.1.0",
"h5py==3.5.0",
@@ -67,8 +73,8 @@ install_requires = [
"ipyparallel==6.2.4",
"ipython==7.12.0",
"ipython_genutils==0.2.0",
- "jupyter-core==4.6.1",
- "jupyter_client==6.1.7",
+ "jupyter-core==4.10.0",
+ "jupyter_client==7.3.4",
"jupyter_console==6.1.0",
"kafka-python==2.0.2",
"karabo_data==0.7.0",
@@ -86,9 +92,9 @@ install_requires = [
"prettytable==0.7.2",
"princess==0.5",
"pypandoc==1.4",
- "python-dateutil==2.8.1",
+ "python-dateutil==2.8.2",
"pyyaml==5.3",
- "pyzmq==19.0.0",
+ "pyzmq==23.2.0",
"requests==2.22.0",
"scikit-learn==0.22.2.post1",
"scipy==1.7.0",
@@ -96,12 +102,13 @@ install_requires = [
"sphinx==1.8.5",
"tabulate==0.8.6",
"traitlets==4.3.3",
- "calibration_client==10.0.0",
+ "xarray==2022.3.0",
+ "EXtra-redu==0.0.5",
]
if "readthedocs.org" not in sys.executable:
install_requires += [
- "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.2.0", # noqa
+ "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.3.0", # noqa
"XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.7.0", # noqa
]
@@ -131,11 +138,11 @@ setup(
entry_points={
"console_scripts": [
"xfel-calibrate = xfel_calibrate.calibrate:run",
+ "xfel-calibrate-repeat = xfel_calibrate.repeat:main",
],
},
cmdclass={
"build": PreInstallCommand,
- "build_ext": build_ext,
},
ext_modules=cythonize(ext_modules, language_level=3),
install_requires=install_requires,
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index 55c99d1f7f0e1caae8eaee6848cb3a1282bb2a8b..3da30383721a0bb9a2c627b07cd101eb24bf63cc 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -22,6 +22,7 @@ from cal_tools.agipdutils import (
make_noisy_adc_mask,
match_asic_borders,
melt_snowy_pixels,
+ cast_array_inplace
)
from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from cal_tools.h5_copy_except import h5_copy_except_paths
@@ -36,22 +37,26 @@ class AgipdCtrl:
ctrl_src: str,
raise_error: bool = True,
):
- """
- Initialize AgipdCondition class to read all required AGIPD parameters.
+ """ Initialize AgipdCondition class to read
+ all required AGIPD parameters.
+ :param run_dc: Run data collection with expected sources
+ to read needed parameters.
:param image_src: H5 source for image data.
:param ctrl_src: H5 source for control (slow) data.
+ :param raise_error: Boolean to raise errors for missing
+ sources and keys.
"""
self.run_dc = run_dc
self.image_src = image_src
self.ctrl_src = ctrl_src
-
self.raise_error = raise_error
def get_num_cells(self) -> Optional[int]:
- """
- :return mem_cells: Number of memory cells.
- return None, if no data available.
+ """Read number of memory cells from fast data.
+
+ :return mem_cells: Number of memory cells
+ return None, if no data available.
"""
cells = np.squeeze(
self.run_dc[
@@ -64,37 +69,29 @@ class AgipdCtrl:
dists = [abs(o - maxcell) for o in options]
return options[np.argmin(dists)]
- def get_acq_rate(self) -> Optional[float]:
- """Get the acquisition rate from said detector module.
-
- If the data is available from the middlelayer FPGA_COMP device,
- then it is retrieved from there.
- If not, the rate is calculated from two different pulses time.
-
- The first entry is deliberately not used, as the detector just began
- operating, and it might have skipped a train.
-
- :return acq_rate: the acquisition rate.
- return None, if not available.
- """
+ def _get_acq_rate_ctrl(self) -> Optional[float]:
+ """Get acquisition (repetition) rate from CONTROL source."""
# Attempt to look for acquisition rate in slow data
rep_rate_src = (
self.ctrl_src, "bunchStructure.repetitionRate.value")
-
if (
rep_rate_src[0] in self.run_dc.all_sources and
rep_rate_src[1] in self.run_dc.keys_for_source(rep_rate_src[0])
):
- # The acquisition rate value is stored in a 1D array of type
- # float.
# It is desired to loose precision here because the usage is
# about bucketing the rate for managing meta-data.
return round(float(self.run_dc[rep_rate_src].as_single_value()), 1)
+ def _get_acq_rate_instr(self) -> Optional[float]:
+ """Get acquisition (repetition rate) from INSTRUMENT source."""
+
+ def _get_acq_rate_instr(self) -> Optional[float]:
+ """Get acquisition (repetition rate) from INSTRUMENT source."""
+
train_pulses = np.squeeze(
self.run_dc[
- self.image_src, "image.pulseId"
- ].drop_empty_trains().train_from_index(0)[1]
+ self.image_src,
+ "image.pulseId"].drop_empty_trains().train_from_index(0)[1]
)
# Compute acquisition rate from fast data
@@ -102,15 +99,35 @@ class AgipdCtrl:
options = {8: 0.5, 4: 1.1, 2: 2.2, 1: 4.5}
return options.get(diff, None)
- def get_gain_setting(
- self,
- creation_time: datetime,
- ) -> Optional[int]:
- """Retrieve Gain setting.
+ def get_acq_rate(self) -> Optional[float]:
+ """Read the acquisition rate for the selected detector module.
+
+ The value is read from CONTROL source e.g.`CONTROL/../MDL/FPGA_COMP`,
+ If key is not available, the rate is calculated from
+ two consecutive pulses of the same trainId.
+
+ :return acq_rate: the acquisition rate.
+ return None, if not available.
+ """
+ acq_rate = self._get_acq_rate_ctrl()
+
+ if acq_rate is not None:
+ return acq_rate
+ # For AGIPD500K this function would produce wrong value (always 4.5)
+ # before W10/2022.
+ # TODO: Confirm leaving this as it is.
+ return self._get_acq_rate_instr()
- If the data is available from the middlelayer FPGA_COMP device,
- then it is retrieved from there.
- If not, the setting is calculated off `setupr` and `patternTypeIndex`
+ def _get_gain_setting_ctrl(self) -> Optional[int]:
+ """Read gain_settings from CONTROL source and gain key."""
+ return int(self.run_dc[self.ctrl_src, "gain"].as_single_value())
+
+ def _get_gain_setting_ctrl_old(self) -> Optional[int]:
+ """Read gain_settings from setupr and patterTypeIndex
+ from old RAW data that is missing `gain.value`.
+
+ If `gain.value` isn't available in MDL source,
+ the setting is calculated from `setupr` and `patternTypeIndex`
gain-setting 1: setupr@dark=8, setupr@slopespc=40
gain-setting 0: setupr@dark=0, setupr@slopespc=32
@@ -119,19 +136,9 @@ class AgipdCtrl:
patternTypeIndex 2: Medium-gain
patternTypeIndex 3: Low-gain
patternTypeIndex 4: SlopesPC
-
- :return: gain setting.
- return 0, if not available.
+ Returns:
+ int: gain_setting value.
"""
- # TODO: remove after fixing get_possible_conditions
- if creation_time and creation_time.replace(tzinfo=None) < parser.parse('2020-01-31'):
- print("Set gain-setting to None for runs taken before 2020-01-31")
- return
-
- if "gain.value" in self.run_dc.keys_for_source(self.ctrl_src):
- return int(
- self.run_dc[self.ctrl_src, "gain"].as_single_value())
-
setupr = self.run_dc[self.ctrl_src, "setupr"].as_single_value()
pattern_type_idx = self.run_dc[
self.ctrl_src, "patternTypeIndex"].as_single_value()
@@ -143,20 +150,50 @@ class AgipdCtrl:
setupr == 40 and pattern_type_idx == 4):
return 1
else:
+ # TODO: Confirm that this can be removed.
if self.raise_error:
raise ValueError(
"Could not derive gain setting from"
" setupr and patternTypeIndex"
)
+ return
+
+ def get_gain_setting(
+ self,
+ creation_time: Optional[datetime] = None,
+ ) -> Optional[int]:
+ """Read Gain setting from CONTROL sources.
+ if key `gain.value` is not available, calculate gain_setting from
+ setupr and patterTypeIndex. If it failed raise ValueError.
+
+ :param creation_time: datetime object for the data creation time.
+ :return: gain setting.
+ return 0, if not available.
+ """
+ # TODO: remove after fixing get_possible_conditions
+ if (
+ creation_time and
+ creation_time.replace(tzinfo=None) < parser.parse('2020-01-31')
+ ):
+ print("Set gain-setting to None for runs taken before 2020-01-31")
+ return
+
+ if "gain.value" in self.run_dc.keys_for_source(self.ctrl_src):
+ return self._get_gain_setting_ctrl()
+ gain_setting = self._get_gain_setting_ctrl_old()
+
+ if gain_setting is not None:
+ return gain_setting
+ else:
+ # TODO: confirm that this can be removed.
print(
- "WARNING: gain_setting is not available "
+ "ERROR: gain_setting is not available "
f"at source {self.ctrl_src}.\nSet gain_setting to 0.")
- # TODO: why return 0 and not None?
return 0
- def get_gain_mode(self) -> AgipdGainMode:
- """Returns the gain mode (adaptive or fixed) from slow data"""
+ def get_gain_mode(self) -> int:
+ """Returns the gain mode (adaptive or fixed) from slow data."""
if (
self.ctrl_src in self.run_dc.all_sources and
@@ -185,7 +222,7 @@ class AgipdCtrl:
:param karabo_id_control: The karabo deviceId for the CONTROL device.
:param module: defaults to module 0
- :return: voltage, a uint16
+ :return: bias voltage
"""
# TODO: Add a breaking fix by passing the source and key through
# get_bias_voltage arguments.
@@ -215,11 +252,11 @@ class AgipdCtrl:
# TODO: Validate if removing this and
# and using NB value for old RAW data.
error = ("ERROR: Unable to read bias_voltage from"
- f" {voltage_src[0]}/{voltage_src[1].replace('.','/')}.")
+ f" {voltage_src[0]}/{voltage_src[1].replace('.','/')}.")
if default_voltage:
print(f"{error} Returning {default_voltage} "
- "as default bias voltage value.")
+ "as default bias voltage value.")
else:
raise ValueError(error)
return default_voltage
@@ -362,6 +399,11 @@ class AgipdCorrections:
self.hg_hard_threshold = 100
self.noisy_adc_threshold = 0.25
self.ff_gain = 1
+ self.actual_photon_energy = 9.2
+
+ # Output parameters
+ self.compress_fields = ['gain', 'mask']
+ self.recast_image_fields = {}
# Shared variables for data and constants
self.shared_dict = []
@@ -382,7 +424,7 @@ class AgipdCorrections:
'zero_orange', 'force_hg_if_below',
'force_mg_if_below', 'mask_noisy_adc',
'melt_snow', 'common_mode', 'mask_zero_std',
- 'low_medium_gap']
+ 'low_medium_gap', 'round_photons']
if set(corr_bools).issubset(tot_corr_bools):
self.corr_bools = corr_bools
@@ -441,8 +483,8 @@ class AgipdCorrections:
data_dict["valid_trains"][:n_valid_trains] = valid_train_ids
# get cell selection for the images in this file
- cm = ( self.cell_sel.CM_NONE if apply_sel_pulses
- else self.cell_sel.CM_PRESEL )
+ cm = (self.cell_sel.CM_NONE if apply_sel_pulses
+ else self.cell_sel.CM_PRESEL)
img_selected = self.cell_sel.get_cells_on_trains(
valid_train_ids, cm=cm)
@@ -458,22 +500,21 @@ class AgipdCorrections:
kw = {
"unstack_pulses": False,
- "pulses": np.nonzero(img_selected),
}
-
# [n_modules, n_imgs, 2, x, y]
raw_data = agipd_comp.get_array("image.data", **kw)[0]
- n_img = raw_data.shape[0]
+ frm_ix = np.flatnonzero(img_selected)
+ n_img = frm_ix.size
data_dict['nImg'][0] = n_img
- data_dict['data'][:n_img] = raw_data[:, 0]
- data_dict['rawgain'][:n_img] = raw_data[:, 1]
+ data_dict['data'][:n_img] = raw_data[frm_ix, 0]
+ data_dict['rawgain'][:n_img] = raw_data[frm_ix, 1]
data_dict['cellId'][:n_img] = agipd_comp.get_array(
- "image.cellId", **kw)[0]
+ "image.cellId", **kw)[0, frm_ix]
data_dict['pulseId'][:n_img] = agipd_comp.get_array(
- "image.pulseId", **kw)[0]
+ "image.pulseId", **kw)[0, frm_ix]
data_dict['trainId'][:n_img] = agipd_comp.get_array(
- "image.trainId", **kw)[0]
+ "image.trainId", **kw)[0, frm_ix]
return n_img
def write_file(self, i_proc, file_name, ofile_name):
@@ -489,18 +530,24 @@ class AgipdCorrections:
agipd_base = f'INSTRUMENT/{self.h5_data_path}/'.format(module_idx)
idx_base = self.h5_index_path.format(module_idx)
data_path = f'{agipd_base}/image'
- data_dict = self.shared_dict[i_proc]
+
+ # Obtain a shallow copy of the pointer map to allow for local
+ # changes in this method.
+ data_dict = self.shared_dict[i_proc].copy()
image_fields = [
'trainId', 'pulseId', 'cellId', 'data', 'gain', 'mask', 'blShift',
]
- compress_fields = ['gain', 'mask']
n_img = data_dict['nImg'][0]
if n_img == 0:
return
trains = data_dict['trainId'][:n_img]
+ # Re-cast fields in-place, i.e. using the same memory region.
+ for field, dtype in self.recast_image_fields.items():
+ data_dict[field] = cast_array_inplace(data_dict[field], dtype)
+
with h5py.File(ofile_name, "w") as outfile:
# Copy any other data from the input file.
# This includes indexes, so it's important that the corrected data
@@ -518,7 +565,7 @@ class AgipdCorrections:
# so it's efficient to examine the file structure.
for field in image_fields:
arr = data_dict[field][:n_img]
- if field in compress_fields:
+ if field in self.compress_fields:
# gain/mask compressed with gzip level 1, but not
# checksummed as we would have to implement this.
kw = dict(
@@ -537,7 +584,7 @@ class AgipdCorrections:
# Write the corrected data
for field in image_fields:
- if field in compress_fields:
+ if field in self.compress_fields:
self._write_compressed_frames(
image_grp[field], data_dict[field][:n_img],
)
@@ -551,8 +598,8 @@ class AgipdCorrections:
in a single thread.
"""
def _compress_frame(i):
- # Equivalent to the HDF5 'shuffle' filter: transpose bytes for better
- # compression.
+ # Equivalent to the HDF5 'shuffle' filter: transpose bytes for
+ # better compression.
shuffled = np.ascontiguousarray(
arr[i].view(np.uint8).reshape((-1, arr.itemsize)).transpose()
)
@@ -860,6 +907,34 @@ class AgipdCorrections:
msk[bidx] |= BadPixels.VALUE_OUT_OF_RANGE
del bidx
+ # Round keV-normalized intensity to photons.
+ if self.corr_bools.get("round_photons"):
+ data /= self.actual_photon_energy
+ np.round(data, out=data)
+
+ # This could also be done before and its mask inverted for
+ # rounding, but the performance difference is negligible.
+ bidx = data < 0
+ data[bidx] = 0
+ msk[bidx] |= BadPixels.VALUE_OUT_OF_RANGE
+ del bidx
+
+ if np.issubdtype(self.recast_image_fields.get('image'), np.integer):
+ # If the image data is meant to be recast to an integer
+ # type, make sure its values are within its bounds.
+
+ type_info = np.iinfo(self.recast_image_data['image'])
+
+ bidx = data < type_info.min
+ data[bidx] = 0
+ msk[bidx] |= BadPixels.VALUE_OUT_OF_RANGE
+ del bidx
+
+ bidx = data > type_info.max
+ data[bidx] = type_info.max
+ msk[bidx] |= BadPixels.VALUE_OUT_OF_RANGE
+ del bidx
+
# Mask entire ADC if they are noise above a threshold
# TODO: Needs clarification if needed,
# the returned arg is not used.
@@ -895,7 +970,7 @@ class AgipdCorrections:
# exclude non valid trains
valid_trains = valid * dc_trains
- return valid_trains[valid_trains!=0]
+ return valid_trains[valid_trains != 0]
def apply_selected_pulses(self, i_proc: int) -> int:
"""Select sharedmem data indices to correct based on selected
@@ -920,27 +995,37 @@ class AgipdCorrections:
if np.all(can_calibrate):
return n_img
+ # Get selected number of images based on
+ # selected pulses to correct
+ n_img_sel = np.count_nonzero(can_calibrate)
+
# Only select data corresponding to selected pulses
# and overwrite data in shared-memory leaving
# the required indices to correct
- data = data_dict['data'][:n_img][can_calibrate]
- rawgain = data_dict['rawgain'][:n_img][can_calibrate]
- cellId = data_dict['cellId'][:n_img][can_calibrate]
- pulseId = data_dict['pulseId'][:n_img][can_calibrate]
- trainId = data_dict['trainId'][:n_img][can_calibrate]
+ array_names = ["data", "rawgain", "cellId", "pulseId", "trainId", "gain"]
- # Overwrite selected number of images based on
- # selected pulses to correct
- n_img = np.count_nonzero(can_calibrate)
+ # if AGIPD in fixed gain mode or melting snow was not requested
+ # `t0_rgain` and `raw_data` will be empty shared_mem arrays
+ is_adaptive = self.gain_mode is AgipdGainMode.ADAPTIVE_GAIN
+ melt_snow = self.corr_bools.get("melt_snow")
+ if (is_adaptive and melt_snow):
+ array_names += ["t0_rgain", "raw_data"]
- data_dict['nImg'][0] = n_img
- data_dict['data'][: n_img] = data
- data_dict['rawgain'][:n_img] = rawgain
- data_dict['cellId'][:n_img] = cellId
- data_dict['pulseId'][:n_img] = pulseId
- data_dict['trainId'][:n_img] = trainId
+ # if baseline correction was not requested
+ # `msk` and `rel_corr` will still be empty shared_mem arrays
+ if any(self.blc_bools):
+ array_names += ["blShift", "msk"]
+ if hasattr(self, "rel_gain"):
+ array_names.append("rel_corr")
- return n_img
+ for name in array_names:
+ arr = data_dict[name][:n_img][can_calibrate]
+ data_dict[name][:n_img_sel] = arr
+
+ # Overwrite number of images
+ data_dict['nImg'][0] = n_img_sel
+
+ return n_img_sel
def copy_and_sanitize_non_cal_data(self, infile, outfile, agipd_base,
idx_base, trains):
@@ -1143,34 +1228,45 @@ class AgipdCorrections:
# calculate median for slopes
pc_high_med = np.nanmedian(pc_high_m, axis=(0, 1))
pc_med_med = np.nanmedian(pc_med_m, axis=(0, 1))
- # calculate median for intercepts:
- pc_high_l_med = np.nanmedian(pc_high_l, axis=(0, 1))
- pc_med_l_med = np.nanmedian(pc_med_l, axis=(0, 1))
-
- # sanitize PC data
- # (it should be done already on the level of constants)
- # In the following loop,
- # replace `nan`s across memory cells with
- # the median value calculated previously.
- # Then, values outside of the valid range (0.8 and 1.2)
- # are fixed to the median value.
- # This is applied for high and medium gain stages
- for i in range(self.max_cells):
- pc_high_m[np.isnan(pc_high_m[..., i]), i] = pc_high_med[i]
- pc_med_m[np.isnan(pc_med_m[..., i]), i] = pc_med_med[i]
-
- pc_high_l[np.isnan(pc_high_l[..., i]), i] = pc_high_l_med[i]
- pc_med_l[np.isnan(pc_med_l[..., i]), i] = pc_med_l_med[i]
-
- pc_high_m[(pc_high_m[..., i] < 0.8 * pc_high_med[i]) |
- (pc_high_m[..., i] > 1.2 * pc_high_med[i]), i] = pc_high_med[i] # noqa
- pc_med_m[(pc_med_m[..., i] < 0.8 * pc_med_med[i]) |
- (pc_med_m[..., i] > 1.2 * pc_med_med[i]), i] = pc_med_med[i] # noqa
-
- pc_high_l[(pc_high_l[..., i] < 0.8 * pc_high_l_med[i]) |
- (pc_high_l[..., i] > 1.2 * pc_high_l_med[i]), i] = pc_high_l_med[i] # noqa
- pc_med_l[(pc_med_l[..., i] < 0.8 * pc_med_l_med[i]) |
- (pc_med_l[..., i] > 1.2 * pc_med_l_med[i]), i] = pc_med_l_med[i] # noqa
+
+ if variant == 0:
+ # calculate median for intercepts:
+ pc_high_l_med = np.nanmedian(pc_high_l, axis=(0, 1))
+ pc_med_l_med = np.nanmedian(pc_med_l, axis=(0, 1))
+
+ # sanitize PC data with CCV variant = 0.
+ # Sanitization is already done for constants
+ # with CCV variant = 1
+ # In the following loop,
+ # replace `nan`s across memory cells with
+ # the median value calculated previously.
+ # Then, values outside of the valid range (0.8 and 1.2)
+ # are fixed to the median value.
+ # This is applied for high and medium gain stages
+ for i in range(self.max_cells):
+ pc_high_m[
+ np.isnan(pc_high_m[..., i]), i] = pc_high_med[i]
+ pc_med_m[
+ np.isnan(pc_med_m[..., i]), i] = pc_med_med[i]
+
+ pc_high_l[
+ np.isnan(pc_high_l[..., i]), i] = pc_high_l_med[i]
+ pc_med_l[
+ np.isnan(pc_med_l[..., i]), i] = pc_med_l_med[i]
+
+ pc_high_m[
+ (pc_high_m[..., i] < 0.8 * pc_high_med[i]) |
+ (pc_high_m[..., i] > 1.2 * pc_high_med[i]), i] = pc_high_med[i] # noqa
+ pc_med_m[
+ (pc_med_m[..., i] < 0.8 * pc_med_med[i]) |
+ (pc_med_m[..., i] > 1.2 * pc_med_med[i]), i] = pc_med_med[i] # noqa
+
+ pc_high_l[
+ (pc_high_l[..., i] < 0.8 * pc_high_l_med[i]) |
+ (pc_high_l[..., i] > 1.2 * pc_high_l_med[i]), i] = pc_high_l_med[i] # noqa
+ pc_med_l[
+ (pc_med_l[..., i] < 0.8 * pc_med_l_med[i]) |
+ (pc_med_l[..., i] > 1.2 * pc_med_l_med[i]), i] = pc_med_l_med[i] # noqa
# ration between HG and MG per pixel per mem cell used
# for rel gain calculation
@@ -1498,34 +1594,33 @@ class CellRange(CellSelection):
return np.tile(self._sel_for_cm(self.flag, self.flag_cm, cm),
len(train_sel))
+
class LitFrameSelection(CellSelection):
"""Selection of detector memery cells indicated as lit frames
by the AgipdLitFrameFinder
"""
- def __init__(self, dev: str, dc: DataCollection, train_ids: List[int],
+ def __init__(self,
+ litfrmdata: 'AgipdLitFrameFinderOffline',
+ train_ids: List[int],
crange: Optional[List[int]] = None,
- energy_threshold: Optional[float] = None):
+ energy_threshold: float = -1000):
"""Initialize lit frame selection
- :param dev: AgipdLitFrameFinder device name
- :param dc: EXtra-data DataCollection of a run
+ :param litfrmdata: AgipdLitFrameFinder output data
:param train_ids: the list of selected trains
:param crange: range parameters of selected cells,
list up to 3 elements
"""
# read AgipdLitFrameFinder data
- self.dev = dev
+ self.dev = litfrmdata.meta.litFrmDev
self.crange = validate_selected_pulses(crange, self.ncell_max)
- self.ethr = energy_threshold
- intr_src = dev + ':output'
- nfrm = dc[intr_src, 'data.nFrame'].ndarray()
- litfrm_train_ids = dc[intr_src, 'data.trainId'].ndarray()
- litfrm = dc[intr_src, 'data.nPulsePerFrame'].ndarray() > 0
- if (energy_threshold != -1000 and
- 'data.energyPerFrame' in dc.keys_for_source(intr_src)):
-
- litfrm &= (dc[intr_src, 'data.energyPerFrame'].ndarray()
- > energy_threshold)
+ self.energy_threshold = energy_threshold
+
+ nfrm = litfrmdata.output.nFrame
+ litfrm_train_ids = litfrmdata.meta.trainId
+ litfrm = litfrmdata.output.nPulsePerFrame > 0
+ if energy_threshold != -1000:
+ litfrm &= litfrmdata.output.energyPerFrame > energy_threshold
# apply range selection
if crange is None:
diff --git a/src/cal_tools/agipdutils.py b/src/cal_tools/agipdutils.py
index 7fd779456838e9e5fc12851349c2596b1c6e9354..6d859edbe2ab7dc7c573e858d41d196c21532664 100644
--- a/src/cal_tools/agipdutils.py
+++ b/src/cal_tools/agipdutils.py
@@ -687,3 +687,39 @@ def melt_snowy_pixels(raw, im, gain, rgain, resolution=None):
snow_mask[k, i * 64:(i + 1) * 64,
j * 64:(j + 1) * 64] = asic_msk
return im, gain, snow_mask
+
+
+def cast_array_inplace(inp, dtype):
+ """Cast an ndarray to a different dtype in place.
+
+ The resulting array will occupy the same memory as the input array,
+ and the cast will most likely make interpretating the buffer content
+ through the input array nonsensical.
+
+ Args:
+ inp (ndarray): Input array to cast, must be contiguous and
+ castable to the target dtype without copy.
+ dtype (DTypeLike): Data type to cast to.
+ """
+
+ # Save shape to recast later
+ orig_shape = inp.shape
+
+ # Create a new view of the input and flatten it in-place. Unlike
+ # inp.reshape(-1) this operation fails if a copy is required.
+ inp = inp.view()
+ inp.shape = inp.size
+
+ # Create a new view with the target dtype and slice it to the number
+ # of elements the input array has. This accounts for smaller dtypes
+ # using less space for the same size.
+ # The output array will be contiguous but not own its data.
+ outp = inp.view(dtype)[:len(inp)]
+
+ # "Copy" over the data, performing the cast.
+ outp[:] = inp
+
+ # Reshape back to the original.
+ outp.shape = orig_shape
+
+ return outp
diff --git a/src/cal_tools/enums.py b/src/cal_tools/enums.py
index a9bfc0182f4972c41259992b1e31cd50b04fba7d..d800eecc83a505613cc21815456e4d3db556ab5c 100644
--- a/src/cal_tools/enums.py
+++ b/src/cal_tools/enums.py
@@ -27,6 +27,7 @@ class BadPixels(IntFlag):
NON_SENSITIVE = 1 << 19 # bit 20
NON_LIN_RESPONSE_REGION = 1 << 20 # bit 21
WRONG_GAIN_VALUE = 1 << 21 # bit 22
+ NON_STANDARD_SIZE = 1 << 22 # bit 23
class SnowResolution(Enum):
""" An Enum specifying how to resolve snowy pixels
diff --git a/src/cal_tools/epix100/epix100lib.py b/src/cal_tools/epix100/epix100lib.py
new file mode 100644
index 0000000000000000000000000000000000000000..ee501960dcacc3d286e364e9e03566b275d0851e
--- /dev/null
+++ b/src/cal_tools/epix100/epix100lib.py
@@ -0,0 +1,26 @@
+import extra_data
+
+
+class epix100Ctrl():
+ def __init__(
+ self,
+ run_dc: extra_data.DataCollection,
+ ctrl_src: str,
+ instrument_src: str,
+ ):
+ """Read epix100 parameters to use later while quering CALCAT.
+ :param run_dir: EXtra-data RunDirectory DataCollection object.
+ :param ctrl_src: CONTROL source for accessing slow data.
+ :param instrument_src: INSTRUMENT source for accessing fast data.
+ """
+ self.run_dc = run_dc
+ self.ctrl_src = ctrl_src
+ self.instrument_src = instrument_src
+
+ def get_integration_time(self):
+ return self.run_dc[
+ self.ctrl_src, 'expTime.value'].as_single_value(reduce_by='first')
+
+ def get_temprature(self):
+ return self.run_dc[
+ self.instrument_src, 'data.backTemp'].ndarray().mean() / 100
diff --git a/src/cal_tools/gotthard2/__init__.py b/src/cal_tools/gotthard2/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/src/cal_tools/gotthard2/gotthard2algs.pyx b/src/cal_tools/gotthard2/gotthard2algs.pyx
new file mode 100644
index 0000000000000000000000000000000000000000..113b5965efea7c813b904d6d549fd9ad3e59b36f
--- /dev/null
+++ b/src/cal_tools/gotthard2/gotthard2algs.pyx
@@ -0,0 +1,59 @@
+from cython cimport boundscheck, cdivision, wraparound
+
+
+@boundscheck(False)
+@wraparound(False)
+def convert_to_10bit(
+ unsigned short[:, :] data,
+ unsigned short[:, :, :] lut,
+ float[:, :] data_10bit,
+):
+ """Convert 12bit RAW data to 10bit data."""
+
+ cdef:
+ unsigned short x, cell, d_10bit, pulse, raw_val, train
+
+ for pulse in range(data.shape[0]):
+ cell = pulse % 2
+ for x in range(data.shape[1]):
+ raw_val = data[pulse, x]
+ d_10bit = lut[x, cell, raw_val]
+ data_10bit[pulse, x] = <float>d_10bit
+
+
+@boundscheck(False)
+@wraparound(False)
+@cdivision(True)
+def correct_train(
+ float[:, :] data,
+ unsigned int[:, :] mask,
+ unsigned char[:, :] gain,
+ float[:, :, :] offset_map,
+ float[:, :, :] gain_map,
+ unsigned int[:, :, :] bpix_map,
+ bint apply_offset = 1,
+ bint apply_gain = 1,
+):
+ """Correct Gotthard2 raw data.
+ 1. Offset correction.
+ 2. Gain correction.
+ 3. Mask badpixels.
+ """
+
+ cdef:
+ float raw_val
+ unsigned short g, x, cell, d_10bit, pulse, train
+
+ for pulse in range(data.shape[0]):
+ cell = pulse % 2
+ for x in range(data.shape[1]):
+ g = gain[pulse, x]
+ if g == 3:
+ g = 2
+ raw_val = data[pulse, x]
+ if apply_offset == 1:
+ raw_val -= offset_map[x, cell, g]
+ if apply_gain == 1:
+ raw_val /= gain_map[x, cell, g]
+ mask[pulse, x] = bpix_map[x, cell, g]
+ data[pulse, x] = raw_val
diff --git a/src/cal_tools/gotthard2/gotthard2lib.py b/src/cal_tools/gotthard2/gotthard2lib.py
new file mode 100644
index 0000000000000000000000000000000000000000..2d4ae564b2b634bb3728d601a8e4a5a33e45ccb1
--- /dev/null
+++ b/src/cal_tools/gotthard2/gotthard2lib.py
@@ -0,0 +1,38 @@
+import extra_data
+
+
+class Gotthard2Ctrl():
+ def __init__(
+ self,
+ run_dc: extra_data.DataCollection,
+ ctrl_src: str,
+ ):
+ """Read slow data.
+ :param run_dir: EXtra-data RunDirectory DataCollection object.
+ :param ctrl_src: Control source name for accessing slow data.
+ """
+ self.run_dc = run_dc
+ self.ctrl_src = ctrl_src
+
+ def get_bias_voltage(self):
+ return self.run_dc[self.ctrl_src, "highVoltageMax"].as_single_value()
+
+ def get_exposure_time(self):
+ return round(
+ self.run_dc[self.ctrl_src, "exposureTime"].as_single_value(), 4)
+
+ def get_exposure_period(self):
+ return round(
+ self.run_dc[self.ctrl_src, "exposurePeriod"].as_single_value(), 4)
+
+ def get_acquisition_rate(self):
+ try:
+ return float(
+ self.run_dc.get_run_value(self.ctrl_src, "acquisitionRate"))
+ except extra_data.PropertyNameError:
+ pass
+
+ def get_single_photon(self):
+ if "singlePhoton.value" in self.run_dc.keys_for_source(self.ctrl_src):
+ return bool(
+ self.run_dc[self.ctrl_src, "singlePhoton"].as_single_value())
diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py
index 88a974c27fe90dc8884b1138e7cbb0ca7f70c9a2..615047957eea9b0f42b232902fbacfe74f0918af 100644
--- a/src/cal_tools/tools.py
+++ b/src/cal_tools/tools.py
@@ -1,5 +1,6 @@
import datetime
import json
+import os
import re
import zlib
from collections import OrderedDict
@@ -19,13 +20,11 @@ import numpy as np
import requests
import yaml
import zmq
-from extra_data import RunDirectory
+from extra_data import H5File, RunDirectory
from iCalibrationDB import ConstantMetaData, Versions
-from metadata_client.metadata_client import MetadataClient
from notebook.notebookapp import list_running_servers
from .ana_tools import save_dict_to_hdf5
-from .restful_config import restful_config
def parse_runs(runs, return_type=str):
@@ -59,14 +58,13 @@ def run_prop_seq_from_path(filename):
def map_seq_files(
- run_dc: "extra_data.DataCollection",
- karabo_id: str,
+ run_folder: Path,
karabo_das: List[str],
sequences: Optional[List[int]] = None,
) -> Tuple[dict, int]:
- """Using a DataCollection from extra-data to read
- available sequence files.
+ """Glob run_folder and match the files based on the selected
+ detectors and sequence numbers.
Returns:
Dict: with karabo_das keys and the corresponding sequence files.
@@ -82,8 +80,8 @@ def map_seq_files(
mapped_files = {kda: [] for kda in karabo_das}
total_files = 0
- for fn in run_dc.select(f"*{karabo_id}*").files:
- fn = Path(fn.filename)
+
+ for fn in run_folder.glob("*.h5"):
if (match := seq_fn_pat.match(fn.name)) is not None:
da = match.group("da")
if da in mapped_files and (
@@ -92,6 +90,10 @@ def map_seq_files(
mapped_files[da].append(fn)
total_files += 1
+ # Return dict with sorted list of sequence files.
+ for k in mapped_files:
+ mapped_files[k].sort()
+
return mapped_files, total_files
@@ -245,55 +247,8 @@ def get_notebook_name():
return environ.get("CAL_NOTEBOOK_NAME", "Unknown Notebook")
-def get_run_info(proposal, run):
- """Return information about run from the MDC
-
- :param proposal: proposal number
- :param run: run number
- :return: dictionary with run information
- """
-
- mdc_config = restful_config['mdc']
- mdc = MetadataClient(
- client_id=mdc_config['user-id'],
- client_secret=mdc_config['user-secret'],
- user_email=mdc_config['user-email'],
- token_url=mdc_config['token-url'],
- refresh_url=mdc_config['refresh-url'],
- auth_url=mdc_config['auth-url'],
- scope=mdc_config['scope'],
- base_api_url=mdc_config['base-api-url'],
- )
-
- mdc_response = mdc.get_proposal_runs(
- proposal_number=proposal, run_number=run)
-
- if mdc_response["success"]:
- return mdc_response
- else: # empty dictionary for wrong proposal or run.
- raise KeyError(mdc_response['app_info'])
-
-
-def creation_date_metadata_client(
- proposal: int, run: int
-) -> datetime.datetime:
- """Get run directory creation date from myMDC using metadata client.
- using method `get_proposal_runs`.
-
- :param proposal: proposal number e.g. 2656 or 900113.
- :param run: run number.
- :return Optional[datetime.datetime]: Run creation date.
- """
-
- run_info = get_run_info(proposal, run)
- return datetime.datetime.strptime(
- run_info['data']['runs'][0]['begin_at'],
- "%Y-%m-%dT%H:%M:%S.%f%z",
- ).astimezone(datetime.timezone.utc)
-
-
def creation_date_file_metadata(
- dc: RunDirectory
+ run_folder: Path,
) -> Optional[datetime.datetime]:
"""Get run directory creation date from
METADATA/CreationDate of the oldest file using EXtra-data.
@@ -302,15 +257,15 @@ def creation_date_file_metadata(
:param dc: EXtra-data DataCollection for the run directory.
:return Optional[datetime.datetime]: Run creation date.
"""
+ md_dict = RunDirectory(run_folder).run_metadata()
- md_dict = dc.run_metadata()
if md_dict["dataFormatVersion"] != "0.5":
- oldest_file = sorted(
- dc.files, key=lambda x: x.metadata()["creationDate"])[0]
+ creation_dates = [
+ H5File(f).run_metadata()["creationDate"]
+ for f in run_folder.glob("*.h5")
+ ]
return datetime.datetime.strptime(
- oldest_file.metadata()["creationDate"],
- "%Y%m%dT%H%M%S%z",
- )
+ min(creation_dates), "%Y%m%dT%H%M%S%z")
else:
print("WARNING: input files contains old datasets. "
"No `METADATA/creationDate` to read.")
@@ -354,10 +309,7 @@ def get_dir_creation_date(directory: Union[str, Path], run: int,
:return: creation datetime for the directory.
"""
- directory = Path(directory)
-
- proposal = int(directory.parent.name[1:])
- directory = directory / 'r{:04d}'.format(run)
+ directory = Path(directory, f'r{run:04d}')
# Validate the availability of the input folder.
# And show a clear error message, if it was not found.
@@ -368,31 +320,40 @@ def get_dir_creation_date(directory: Union[str, Path], run: int,
"- Failed to read creation time, wrong input folder",
directory) from e
- try:
- return creation_date_metadata_client(proposal, run)
- except Exception as e:
- if verbosity > 0:
- print(e)
-
cdate = creation_date_train_timestamp(dc)
if cdate is not None:
# Exposing the method used for reading the creation_date.
print("Reading creation_date from input files metadata"
- " `METADATA/creationDate`")
+ " `INDEX/timestamp`")
else: # It's an older dataset.
print("Reading creation_date from last modification data "
"for the oldest input file.")
cdate = datetime.datetime.fromtimestamp(
sorted(
- [Path(f.filename) for f in dc.files],
- key=path.getmtime
+ directory.glob("*.h5"), key=path.getmtime,
)[0].stat().st_mtime,
tz=datetime.timezone.utc,
)
return cdate
+def calcat_creation_time(
+ in_folder: Path,
+ run: str,
+ creation_time: Optional[str] = "",
+ ) -> datetime.datetime:
+ """Return the creation time to use with CALCAT."""
+ # Run's creation time:
+ if creation_time:
+ creation_time = datetime.datetime.strptime(
+ creation_time,
+ '%Y-%m-%d %H:%M:%S').astimezone(tz=datetime.timezone.utc)
+ else:
+ creation_time = get_dir_creation_date(in_folder, run)
+ return creation_time
+
+
def _init_metadata(constant: 'iCalibrationDB.calibration_constant',
condition: 'iCalibrationDB.detector_conditions',
creation_time: Optional[str] = None
@@ -472,13 +433,17 @@ def save_const_to_h5(db_module: str, karabo_id: str,
def get_random_db_interface(cal_db_interface):
+ """Return interface to calibration DB with
+ random (with given range) port.
"""
- Return interface to calibration DB with random (with given range) port.
- """
+ # Initialize the random generator with a random seed value,
+ # in case the function was executed within a multiprocessing pool.
+ np.random.seed()
if "#" in cal_db_interface:
prot, serv, ran = cal_db_interface.split(":")
r1, r2 = ran.split("#")
- return ":".join([prot, serv, str(np.random.randint(int(r1), int(r2)))])
+ return ":".join(
+ [prot, serv, str(np.random.randint(int(r1), int(r2)))])
return cal_db_interface
@@ -870,6 +835,9 @@ class CalibrationMetadata(dict):
else:
print(f"Warning: existing {self._yaml_fn} is malformed, "
"will be overwritten")
+ @property
+ def filename(self):
+ return self._yaml_fn
def save(self):
with self._yaml_fn.open("w") as fd:
@@ -880,6 +848,73 @@ class CalibrationMetadata(dict):
yaml.safe_dump(dict(self), fd)
+def save_constant_metadata(
+ retrieved_constants: dict,
+ mdata: ConstantMetaData,
+ constant_name: str,
+ ):
+ """Save constant metadata to the input meta data dictionary.
+ The constant's metadata stored are file path, dataset name,
+ creation time, and physical detector unit name.
+
+ :param retrieved_constants: A dictionary to store the metadata for
+ the retrieved constant.
+ :param mdata: A ConstantMetaData object after retrieving trying
+ to retrieve a constant with get_from_db().
+ :param constant_name: String for constant name to be used as a key.
+ :param constants_key: The key name when all constants metadata
+ will be stored.
+ """
+
+ mdata_const = mdata.calibration_constant_version
+ const_mdata = retrieved_constants[constant_name] = dict()
+ # check if constant was successfully retrieved.
+ if mdata.comm_db_success:
+ const_mdata["file-path"] = (
+ f"{mdata_const.hdf5path}" f"{mdata_const.filename}"
+ )
+ const_mdata["dataset-name"] = mdata_const.h5path
+ const_mdata["creation-time"] = mdata_const.begin_at
+ else:
+ const_mdata["file-path"] = None
+ const_mdata["creation-time"] = None
+
+
+def load_specified_constants(
+ retrieved_constants: dict,
+ empty_constants: Optional[dict] = None,
+ ) -> Tuple[dict, dict]:
+ """Load constant data from metadata in the
+ retrieved_constants dictionary.
+
+ :param retrieved_constants: A dict. with the constant filepaths and
+ dataset-name to read the constant data arrays.
+ {
+ 'Constant Name': {
+ 'file-path': '/gpfs/.../*.h5',
+ 'dataset-name': '/module_name/...',
+ 'creation-time': str(datetime),},
+ }
+ :param empty_constants: A dict of constant names keys and
+ the empty constant array to use in case of not non-retrieved constants.
+ :return constant_data: A dict of constant names keys and their data.
+ """
+ const_data = dict()
+ when = dict()
+
+ for cname, mdata in retrieved_constants.items():
+ const_data[cname] = dict()
+ when[cname] = mdata["creation-time"]
+ if when[cname]:
+ with h5py.File(mdata["file-path"], "r") as cf:
+ const_data[cname] = np.copy(
+ cf[f"{mdata['dataset-name']}/data"])
+ else:
+ const_data[cname] = (
+ empty_constants[cname] if empty_constants else None)
+ return const_data, when
+
+
def write_compressed_frames(
arr: np.ndarray,
ofile: h5py.File,
diff --git a/src/xfel_calibrate/calibrate.py b/src/xfel_calibrate/calibrate.py
index 526024fc8e831eb75f84b574cc581733d527808a..704b18e20c35e9990f0c6901be8b041d2e3287ae 100755
--- a/src/xfel_calibrate/calibrate.py
+++ b/src/xfel_calibrate/calibrate.py
@@ -38,7 +38,6 @@ from .settings import (
launcher_command,
python_path,
sprof,
- temp_path,
try_report_to_output,
)
@@ -181,11 +180,11 @@ def flatten_list(l):
return ''
-def create_finalize_script(fmt_args, temp_path, job_list) -> str:
+def create_finalize_script(fmt_args, cal_work_dir, job_list) -> str:
"""
Create a finalize script to produce output report
:param fmt_args: Dictionary of fmt arguments
- :param temp_path: Path to temporary folder to run slurm job
+ :param cal_work_dir: Path to temporary folder to run slurm job
:param job_list: List of slurm jobs
:return: The path of the created script
"""
@@ -196,7 +195,7 @@ def create_finalize_script(fmt_args, temp_path, job_list) -> str:
finalize(joblist={{joblist}},
finaljob=os.environ.get('SLURM_JOB_ID', ''),
- run_path='{{run_path}}',
+ cal_work_dir='{{cal_work_dir}}',
out_path='{{out_path}}',
version='{{version}}',
title='{{title}}',
@@ -209,7 +208,7 @@ def create_finalize_script(fmt_args, temp_path, job_list) -> str:
""")
fmt_args['joblist'] = job_list
- f_name = os.path.join(temp_path, "finalize.py")
+ f_name = os.path.join(cal_work_dir, "finalize.py")
with open(f_name, "w") as finfile:
finfile.write(textwrap.dedent(tmpl.render(**fmt_args)))
@@ -221,8 +220,10 @@ def create_finalize_script(fmt_args, temp_path, job_list) -> str:
def run_finalize(
- fmt_args, temp_path, job_list, sequential=False, partition="exfel"):
- finalize_script = create_finalize_script(fmt_args, temp_path, job_list)
+ fmt_args, cal_work_dir, job_list, sequential=False, partition=None):
+ if partition is None:
+ partition = "exfel"
+ finalize_script = create_finalize_script(fmt_args, cal_work_dir, job_list)
cmd = []
if not sequential:
@@ -230,7 +231,7 @@ def run_finalize(
'sbatch',
'--parsable',
'--requeue',
- '--output', f'{temp_path}/slurm-%j.out',
+ '--output', f'{cal_work_dir}/slurm-%j.out',
'--open-mode=append', # So we can see if it's preempted & requeued
'--job-name', 'xfel-cal-finalize',
'--time', finalize_time_limit,
@@ -242,7 +243,7 @@ def run_finalize(
cmd += [
os.path.join(PKG_DIR, "bin", "slurm_finalize.sh"), # path to helper sh
sys.executable, # Python with calibration machinery installed
- temp_path,
+ cal_work_dir,
finalize_script,
fmt_args['report_to']
]
@@ -255,20 +256,6 @@ def run_finalize(
return jobid
-def save_executed_command(run_tmp_path, version, argv):
- """
- Create a file with string used to execute `xfel_calibrate`
-
- :param run_tmp_path: path to temporary directory for running job outputs
- :param version: git version of the pycalibration package
- """
-
- f_name = os.path.join(run_tmp_path, "run_calibrate.sh")
- with open(f_name, "w") as finfile:
- finfile.write(f'# pycalibration version: {version}\n')
- finfile.write(shlex.join(argv))
-
-
class SlurmOptions:
def __init__(
self, job_name=None, nice=None, mem=None, partition=None, reservation=None,
@@ -289,17 +276,16 @@ class SlurmOptions:
return ['--reservation', self.reservation]
return ['--partition', self.partition or sprof]
- def get_launcher_command(self, temp_path, after_ok=(), after_any=()) -> List[str]:
+ def get_launcher_command(self, log_dir, after_ok=(), after_any=()) -> List[str]:
"""
Return a slurm launcher command
- :param args: Command line arguments
- :param temp_path: Temporary path to run job
+ :param log_dir: Where Slurm .out log files should go
:param after_ok: A list of jobs which must succeed first
:param after_any: A list of jobs which must finish first, but may fail
:return: List of commands and parameters to be used by subprocess
"""
- launcher_slurm = launcher_command.format(temp_path=temp_path).split()
+ launcher_slurm = launcher_command.format(temp_path=log_dir).split()
launcher_slurm += self.get_partition_or_reservation()
@@ -338,7 +324,7 @@ def remove_duplications(l) -> list:
def prepare_job(
- temp_path: str, nb, nb_path: Path, args: dict, cparm=None, cval=None,
+ cal_work_dir: Path, nb, nb_path: Path, args: dict, cparm=None, cval=None,
cluster_cores=8, show_title=True,
) -> 'JobArgs':
if cparm is not None:
@@ -367,8 +353,7 @@ def prepare_job(
set_figure_format(new_nb, args["vector_figs"])
new_name = f"{nb_path.stem}__{cparm}__{suffix}.ipynb"
- nbpath = os.path.join(temp_path, new_name)
- nbformat.write(new_nb, nbpath)
+ nbformat.write(new_nb, cal_work_dir / new_name)
return JobArgs([
"./pycalib-run-nb.sh", # ./ allows this to run directly
@@ -491,12 +476,11 @@ class JobChain:
return errors
-def make_par_table(parms, run_tmp_path: str):
+def make_par_table(parms):
"""
- Create a table with input parameters of the notebook
+ Create a RST table with input parameters of the notebook
:param parms: parameters of the notebook
- :param run_tmp_path: path to temporary directory for running job outputs
"""
# Add space in long strings without line breakers ` ,-/` to
@@ -552,9 +536,7 @@ def make_par_table(parms, run_tmp_path: str):
\end{longtable}
''')
- f_name = os.path.join(run_tmp_path, "InputParameters.rst")
- with open(f_name, "w") as finfile:
- finfile.write(textwrap.dedent(tmpl.render(p=col_type, lines=l_parms)))
+ return textwrap.dedent(tmpl.render(p=col_type, lines=l_parms))
def run(argv=None):
@@ -609,23 +591,6 @@ def run(argv=None):
if args.get("cluster_profile") == default_params_by_name["cluster_profile"]:
args['cluster_profile'] = "slurm_prof_{}".format(run_uuid)
- # create a temporary output directory to work in
- run_tmp_path = os.path.join(
- temp_path, f"slurm_out_{nb_details.detector}_{nb_details.caltype}_{run_uuid}"
- )
- os.makedirs(run_tmp_path)
-
- # Write all input parameters to rst file to be included to final report
- parms = parameter_values(nb_details.default_params, **args)
- make_par_table(parms, run_tmp_path)
- # And save the invocation of this script itself
- save_executed_command(run_tmp_path, version, argv)
-
- # Copy the bash script which will be used to run notebooks
- shutil.copy2(
- os.path.join(PKG_DIR, "bin", "slurm_calibrate.sh"),
- os.path.join(run_tmp_path, "pycalib-run-nb.sh")
- )
# wait on all jobs to run and then finalize the run by creating a report from the notebooks
out_path = Path(default_report_path) / nb_details.detector / nb_details.caltype / datetime.now().isoformat()
@@ -652,6 +617,27 @@ def run(argv=None):
print(f"report_to path contained no path, saving report in '{out_path}'")
report_to = out_path / report_to
+ workdir_name = f"slurm_out_{nb_details.detector}_{nb_details.caltype}_{run_uuid}"
+ if report_to:
+ cal_work_dir = report_to.parent / workdir_name
+ else:
+ cal_work_dir = out_path / workdir_name
+ cal_work_dir.mkdir(parents=True)
+
+ # Write all input parameters to rst file to be included to final report
+ parms = parameter_values(nb_details.default_params, **args)
+ (cal_work_dir / "InputParameters.rst").write_text(make_par_table(parms))
+ # And save the invocation of this script itself
+ (cal_work_dir / "run_calibrate.sh").write_text(
+ f'# pycalibration version: {version}\n' + shlex.join(argv)
+ )
+
+ # Copy the bash script which will be used to run notebooks
+ shutil.copy2(
+ os.path.join(PKG_DIR, "bin", "slurm_calibrate.sh"),
+ cal_work_dir / "pycalib-run-nb.sh"
+ )
+
if nb_details.user_venv:
print("Using specified venv:", nb_details.user_venv)
python_exe = str(nb_details.user_venv / 'bin' / 'python')
@@ -659,7 +645,7 @@ def run(argv=None):
python_exe = python_path
# Write metadata about calibration job to output folder
- metadata = cal_tools.tools.CalibrationMetadata(run_tmp_path, new=True)
+ metadata = cal_tools.tools.CalibrationMetadata(cal_work_dir, new=True)
parm_subdict = metadata.setdefault("calibration-configurations", {})
for p in parms:
@@ -691,7 +677,7 @@ def run(argv=None):
# Record installed Python packages for reproducing the environment
if not args['skip_env_freeze']:
- with open(os.path.join(run_tmp_path, 'requirements.txt'), 'wb') as f:
+ with (cal_work_dir / 'requirements.txt').open('wb') as f:
check_call([python_exe, '-m', 'pip', 'freeze'], stdout=f)
folder = get_par_attr(parms, 'in_folder', 'value', '')
@@ -707,14 +693,14 @@ def run(argv=None):
for pre_notebook_path in nb_details.pre_paths:
lead_nb = nbformat.read(pre_notebook_path, as_version=4)
pre_jobs.append(prepare_job(
- run_tmp_path, lead_nb, pre_notebook_path, args,
+ cal_work_dir, lead_nb, pre_notebook_path, args,
cluster_cores=cluster_cores
))
main_jobs = []
if concurrency_par is None:
main_jobs.append(prepare_job(
- run_tmp_path, nb,
+ cal_work_dir, nb,
notebook_path, args,
cluster_cores=cluster_cores,
))
@@ -768,7 +754,7 @@ def run(argv=None):
cval = [cval, ] if not isinstance(cval, list) and cvtype is list else cval
main_jobs.append(prepare_job(
- run_tmp_path, nb, notebook_path, args,
+ cal_work_dir, nb, notebook_path, args,
concurrency_par, cval,
cluster_cores=cluster_cores,
show_title=show_title,
@@ -779,7 +765,7 @@ def run(argv=None):
for i, dep_notebook_path in enumerate(nb_details.dep_paths):
dep_nb = nbformat.read(dep_notebook_path, as_version=4)
dep_jobs.append(prepare_job(
- run_tmp_path, dep_nb, dep_notebook_path, args,
+ cal_work_dir, dep_nb, dep_notebook_path, args,
cluster_cores=cluster_cores,
))
@@ -787,24 +773,23 @@ def run(argv=None):
Step(pre_jobs),
Step(main_jobs),
Step(dep_jobs, after_error=True)
- ], Path(run_tmp_path), python_exe)
+ ], Path(cal_work_dir), python_exe)
# Save information about jobs for reproducibility
job_chain.save()
if args['prepare_only']:
- # FIXME: Clean up where this file goes when.
- # When the jobs run, it is copied by finalize.py
- metadata.save_copy(Path(run_tmp_path))
-
print("Files prepared, not executing now (--prepare-only option).")
print("To execute the notebooks, run:")
rpt_opts = ''
if nb_details.user_venv is not None:
rpt_opts = f'--python {python_exe}'
- print(f" python -m xfel_calibrate.repeat {run_tmp_path} {rpt_opts}")
+ print(f" python -m xfel_calibrate.repeat {cal_work_dir} {rpt_opts}")
return
+ print("Calibration work directory (including Slurm .out files):")
+ print(" ", cal_work_dir)
+
submission_time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
# Launch the calibration work
@@ -824,7 +809,7 @@ def run(argv=None):
))
errors = False
- fmt_args = {'run_path': run_tmp_path,
+ fmt_args = {'cal_work_dir': cal_work_dir,
'out_path': out_path,
'version': version,
'title': title,
@@ -837,7 +822,7 @@ def run(argv=None):
joblist.append(run_finalize(
fmt_args=fmt_args,
- temp_path=run_tmp_path,
+ cal_work_dir=cal_work_dir,
job_list=joblist,
sequential=args["no_cluster_job"],
partition=args["slurm_partition"] or "exfel",
diff --git a/src/xfel_calibrate/finalize.py b/src/xfel_calibrate/finalize.py
index 7eabeb96ba531c562b6f667a05186b4fb8f6b1d1..0911f0afb7ac63ab9643009b33527da6e666536a 100644
--- a/src/xfel_calibrate/finalize.py
+++ b/src/xfel_calibrate/finalize.py
@@ -1,16 +1,13 @@
import re
import sys
from datetime import datetime
-from glob import glob
from importlib.machinery import SourceFileLoader
-from os import chdir, listdir, makedirs, path, stat
-from os.path import isdir, isfile, splitext
+from os import chdir, listdir, path
from pathlib import Path
from shutil import copy, copytree, move, rmtree
from subprocess import CalledProcessError, check_call, check_output
from tempfile import TemporaryDirectory
from textwrap import dedent
-from time import sleep
from typing import Dict, List
import tabulate
@@ -39,59 +36,55 @@ def natural_keys(text):
return [atoi(c) for c in re.split(r'(\d+)', text)]
-def combine_report(run_path, calibration):
- sphinx_path = "{}/sphinx_rep".format(path.abspath(run_path))
+def combine_report(cal_work_dir):
+ sphinx_path = Path(cal_work_dir, "sphinx_rep").resolve()
# if the finalize job was preempted or requeued,
# while building the report.
- if isdir(sphinx_path):
+ if sphinx_path.is_dir():
rmtree(sphinx_path)
- makedirs(sphinx_path)
- direntries = listdir(run_path)
+ sphinx_path.mkdir(parents=True)
+ direntries = listdir(cal_work_dir)
direntries.sort(key=natural_keys)
for entry in direntries:
-
- if isfile("{}/{}".format(run_path, entry)):
- name, ext = splitext("{}".format(entry))
-
- if ext == ".rst":
- comps = name.split("__")
- if len(comps) >= 3:
- group, name_param, conc_param = "_".join(comps[:-2]), \
- comps[-2], comps[-1]
+ entry = Path(cal_work_dir, entry)
+ if entry.suffix == '.rst' and entry.is_file():
+ comps = entry.stem.split("__")
+ if len(comps) >= 3:
+ group, name_param, conc_param = "_".join(comps[:-2]), \
+ comps[-2], comps[-1]
+ else:
+ group, name_param, conc_param = comps[0], "None", "None"
+
+ rst_content = entry.read_text('utf-8')
+
+ # If this is part of a group (same notebook run on different
+ # sequences or modules), replace its title so each section has
+ # a meaningful heading:
+ if conc_param != 'None':
+ # Match a title at the start of the file with === underline
+ m = re.match(r"\n*(.{3,})\n={3,}\n\n", rst_content)
+ if m:
+ old_title = m.group(1)
+ _, title_end = m.span()
+ rst_content = rst_content[title_end:]
else:
- group, name_param, conc_param = comps[0], "None", "None"
-
- with open("{}/{}.rst".format(sphinx_path, group),
- "a") as gfile:
- if conc_param != "None":
- title = "{}, {} = {}".format(calibration, name_param,
- conc_param)
- gfile.write(title + "\n")
- gfile.write("=" * len(title) + "\n")
- gfile.write("\n")
- with open("{}/{}".format(run_path, entry), "r") as ifile:
- skip_next = False
- for line in ifile.readlines():
- if skip_next:
- skip_next = False
- continue
- if conc_param == "None":
- gfile.write(line)
- elif " ".join(calibration.split()) != " ".join(
- line.split()):
- gfile.write(line)
- else:
- skip_next = True
-
- gfile.write("\n\n")
- if isdir("{}/{}".format(run_path, entry)):
- copytree("{}/{}".format(run_path, entry),
- "{}/{}".format(sphinx_path, entry))
+ old_title = 'Processing' # Generic fallback
+ title = f"{old_title}, {name_param} = {conc_param}"
+ rst_content = '\n'.join([
+ title, '=' * len(title), '', rst_content
+ ])
+
+ with (sphinx_path / f"{group}.rst").open("a", encoding="utf-8") as f:
+ f.write(rst_content)
+ f.write("\n\n")
+
+ elif entry.is_dir():
+ copytree(entry, sphinx_path / entry.name)
return sphinx_path
-def prepare_plots(run_path, threshold=1000000):
+def prepare_plots(cal_work_dir: Path, threshold=1_000_000):
"""
Convert svg file to pdf or png to be used for latex
@@ -102,36 +95,25 @@ def prepare_plots(run_path, threshold=1000000):
The links in the rst files are adapted accordingly to the
converted image files.
- :param run_path: Run path of the slurm job
+ :param cal_work_dir: Run path of the slurm job
:param threshold: Max svg file size (in bytes) to be converted to pdf
"""
print('Convert svg to pdf and png')
- run_path = path.abspath(run_path)
- rst_files = glob('{}/*rst'.format(run_path))
- for rst_file in rst_files:
- rst_file_name = path.basename(rst_file)
- rst_file_name = path.splitext(rst_file_name)[0]
+ for rst_file in cal_work_dir.glob("*.rst"):
- svg_files = glob(
- '{}/{}_files/*svg'.format(run_path, rst_file_name))
- for f_path in svg_files:
- f_name = path.basename(f_path)
- f_name = path.splitext(f_name)[0]
-
- if (stat(f_path)).st_size < threshold:
- check_call(["svg2pdf", "{}".format(f_path)], shell=False)
+ for f_path in (cal_work_dir / f'{rst_file.stem}_files').glob('*.svg'):
+ if f_path.stat().st_size < threshold:
+ check_call(["svg2pdf", str(f_path)], shell=False)
new_ext = 'pdf'
else:
- check_call(["convert", "{}".format(f_path),
- "{}.png".format(f_name)], shell=False)
+ check_call(["convert", str(f_path),
+ str(f_path.with_suffix('.png'))], shell=False)
new_ext = 'png'
- check_call(["sed",
- "-i",
- "s/{}.svg/{}.{}/g".format(f_name, f_name, new_ext),
- rst_file],
- shell=False)
+ check_call([
+ "sed", "-i", f"s/{f_path.name}/{f_path.stem}.{new_ext}/g", rst_file
+ ], shell=False)
def get_job_info(jobs: List[str], fmt: List[str]) -> List[List[str]]:
@@ -162,12 +144,12 @@ def get_job_info(jobs: List[str], fmt: List[str]) -> List[List[str]]:
return [job_info[job] for job in jobs]
-def make_timing_summary(run_path: Path, job_times: List[List[str]],
+def make_timing_summary(cal_work_dir: Path, job_times: List[List[str]],
job_time_fmt: List[str], pipeline_times: Dict[str, str]):
"""
Create an rst file with timing summary of executed notebooks
- :param run_path: Run path of the slurm job
+ :param cal_work_dir: Run path of the slurm job
:param job_times: List of job information as returned by get_job_info
:param job_time_fmt: List of headers to use for job_times
:param pipeline_times: Dictionary of pipeline step -> timestamp
@@ -199,7 +181,7 @@ def make_timing_summary(run_path: Path, job_times: List[List[str]],
["Report compilation", pipeline_times["report-compilation-time"]],
]
- with (run_path / "timing_summary.rst").open("w+") as fd:
+ with (cal_work_dir / "timing_summary.rst").open("w+") as fd:
time_table = tabulate.tabulate(time_vals, tablefmt="latex",
headers=["Processing step", "Timestamp"])
@@ -211,7 +193,7 @@ def make_timing_summary(run_path: Path, job_times: List[List[str]],
fd.write(dedent(job_tbl_tmpl.render(job_table=job_table.split("\n"))))
-def make_report(run_path: Path, tmp_path: Path, project: str,
+def make_report(run_path: Path, cal_work_dir: Path, project: str,
author: str, version: str, report_to: Path):
"""
Create calibration report (pdf file)
@@ -220,7 +202,7 @@ def make_report(run_path: Path, tmp_path: Path, project: str,
jupyter-notebooks.
:param run_path: Path to sphinx run directory
- :param tmp_path: Run path of the slurm job
+ :param cal_work_dir: Run path of the slurm job
:param project: Project title
:param author: Author of the notebook
:param version: Version of the notebook
@@ -336,7 +318,7 @@ def make_report(run_path: Path, tmp_path: Path, project: str,
copy(run_path / "_build" / "latex" / f"{report_name}.pdf", report_dir)
# Remove folders with figures and sphinx files.
- for tmp_subdir in tmp_path.iterdir():
+ for tmp_subdir in cal_work_dir.iterdir():
if tmp_subdir.is_dir():
print(f"Removing temporary subdir: {tmp_subdir}")
rmtree(tmp_subdir)
@@ -388,15 +370,15 @@ def tex_escape(text):
return regex.sub(lambda match: conv[match.group()], text)
-def finalize(joblist, finaljob, run_path, out_path, version, title, author, report_to, data_path='Unknown',
+def finalize(joblist, finaljob, cal_work_dir, out_path, version, title, author, report_to, data_path='Unknown',
request_time='', submission_time=''):
- run_path = Path(run_path)
+ cal_work_dir = Path(cal_work_dir)
out_path = Path(out_path)
# Archiving files in slurm_tmp
if finaljob:
joblist.append(str(finaljob))
- metadata = cal_tools.tools.CalibrationMetadata(run_path)
+ metadata = cal_tools.tools.CalibrationMetadata(cal_work_dir)
job_time_fmt = 'JobID,Start,End,Elapsed,Suspended,State'.split(',')
job_time_summary = get_job_info(joblist, job_time_fmt)
@@ -405,7 +387,7 @@ def finalize(joblist, finaljob, run_path, out_path, version, title, author, repo
"submission-time": submission_time,
"report-compilation-time": datetime.now().strftime("%Y-%m-%dT%H:%M:%S"),
}
- make_timing_summary(run_path, job_time_summary, job_time_fmt, pipeline_time_summary)
+ make_timing_summary(cal_work_dir, job_time_summary, job_time_fmt, pipeline_time_summary)
metadata.update(
{
"runtime-summary": {
@@ -419,21 +401,18 @@ def finalize(joblist, finaljob, run_path, out_path, version, title, author, repo
if report_to:
report_to = Path(report_to)
- prepare_plots(run_path)
+ prepare_plots(cal_work_dir)
- sphinx_path = combine_report(run_path, title)
+ sphinx_path = combine_report(cal_work_dir)
make_titlepage(sphinx_path, title, data_path, version)
make_report(
Path(sphinx_path),
- run_path,
+ cal_work_dir,
title,
author,
version,
report_to,
)
- # Store the contents of the Slurm working directory next to the report.
- # This contains elements needed for reproducibility.
- slurm_archive_dir = report_to.parent / f"slurm_out_{report_to.name}"
det = metadata['calibration-configurations'].get('karabo-id', report_to.name)
else:
try:
@@ -444,14 +423,8 @@ def finalize(joblist, finaljob, run_path, out_path, version, title, author, repo
from hashlib import sha1
det = sha1(''.join(joblist).encode('ascii')).hexdigest()[:8]
- # If there is no report location, simply move the slurm_out_
- # directory to the output.
- slurm_archive_dir = Path(out_path) / f"slurm_out_{det}"
-
- print(f"Moving temporary files to final location: {slurm_archive_dir}")
- move(str(run_path), str(slurm_archive_dir)) # Needs str until Python 3.9
+ md_path = cal_work_dir / "calibration_metadata.yml"
- md_path = slurm_archive_dir / "calibration_metadata.yml"
# Notebooks should have a karabo_id parameter, which we'll use to make a
# unique name like calibration_metadata_MID_DET_AGIPD1M-1.yml in the output
# folder. In case they don't, fall back to a name like the report.
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index a26c4afdf8ea3da72a9de9300f0796b016ad517a..6c7f5d5e7353c7f20c6b342047055ba2e3e55528 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -52,15 +52,6 @@ notebooks = {
"cluster cores": 1},
},
},
- "AGIPD64K": {
- "DARK": {
- "notebook":
- "notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb",
- "concurrency": {"parameter": None,
- "default concurrency": None,
- "cluster cores": 4},
- },
- },
"LPD": {
"DARK": {
"notebook": "notebooks/LPD/LPDChar_Darks_NBC.ipynb",
@@ -85,6 +76,8 @@ notebooks = {
"cluster cores": 8},
},
"CORRECT": {
+ "pre_notebooks": [
+ "notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb"],
"notebook": "notebooks/LPD/LPD_Correct_Fast.ipynb",
"concurrency": {"parameter": "sequences",
"default concurrency": [-1],
@@ -119,6 +112,7 @@ notebooks = {
"cluster cores": 32},
},
"CORRECT": {
+ "pre_notebooks": ["notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb"],
"notebook": "notebooks/pnCCD/Correct_pnCCD_NBC.ipynb",
"concurrency": {"parameter": "sequences",
"default concurrency": [-1],
@@ -183,6 +177,8 @@ notebooks = {
"cluster cores": 4},
},
"CORRECT": {
+ "pre_notebooks": [
+ "notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb"], # noqa
"notebook":
"notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb",
"concurrency": {"parameter": "sequences",
@@ -191,15 +187,35 @@ notebooks = {
"cluster cores": 14},
},
},
+ "GOTTHARD2": {
+ "CORRECT": {
+ "pre_notebooks": [
+ "notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb"], # noqa
+ "notebook":
+ "notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb",
+ "concurrency": {"parameter": "sequences",
+ "default concurrency": [-1],
+ "use function": "balance_sequences",
+ "cluster cores": 16},
+ },
+ "DARK": {
+ "notebook":
+ "notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb",
+ "concurrency": {"parameter": "karabo_da",
+ "default concurrency": list(range(2)),
+ "cluster cores": 4},
+ },
+ },
"EPIX100": {
"DARK": {
- "notebook": "notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb",
+ "notebook": "notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb", # noqa
"concurrency": {"parameter": None,
"default concurrency": None,
"cluster cores": 4},
},
"CORRECT": {
+ "pre_notebooks": ["notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb"],
"notebook": "notebooks/ePix100/Correction_ePix100_NBC.ipynb",
"concurrency": {"parameter": "sequences",
"default concurrency": [-1],
diff --git a/src/xfel_calibrate/repeat.py b/src/xfel_calibrate/repeat.py
index 06e091f16553042a7cf3bf1c17d964dc24b8541e..9992ac59be047e7ea3f042c26fbbbdf665fc44a9 100644
--- a/src/xfel_calibrate/repeat.py
+++ b/src/xfel_calibrate/repeat.py
@@ -1,10 +1,12 @@
import argparse
+import os
import shutil
import sys
from datetime import datetime
from pathlib import Path
import nbformat
+from env_cache import EnvsManager, PyenvEnvMaker
from nbparameterise import extract_parameters, parameter_values, replace_definitions
from cal_tools.tools import CalibrationMetadata
@@ -13,6 +15,51 @@ from .calibrate import (
)
from .settings import temp_path
+# This function is copied and modified from Python 3.8.10
+# Copyright © 2001-2022 Python Software Foundation; All Rights Reserved
+# Used under the PSF license - https://docs.python.org/3/license.html
+def copytree_no_metadata(src, dst, ignore):
+ """Copy directory contents, without permissions or extended attributes
+
+ Copying the GPFS ACLs can cause problems when creating new files in
+ the directory, so we avoid that. shutil.copytree can skip copying file
+ metadata, but not directory metadata:
+ https://github.com/python/cpython/issues/89721
+
+ This is also simplified by removing options we're not using.
+ """
+ with os.scandir(src) as itr:
+ entries = list(itr)
+ if ignore is not None:
+ ignored_names = ignore(os.fspath(src), [x.name for x in entries])
+ else:
+ ignored_names = set()
+
+ os.makedirs(dst)
+ errors = []
+
+ for srcentry in entries:
+ if srcentry.name in ignored_names:
+ continue
+ srcname = os.path.join(src, srcentry.name)
+ dstname = os.path.join(dst, srcentry.name)
+ try:
+ if srcentry.is_dir():
+ copytree_no_metadata(srcentry, dstname, ignore)
+ else:
+ # Will raise a SpecialFileError for unsupported file types
+ shutil.copyfile(srcentry, dstname)
+ # catch the Error from the recursive copytree so that we can
+ # continue with other files
+ except shutil.Error as err:
+ errors.extend(err.args[0])
+ except OSError as why:
+ errors.append((srcname, dstname, str(why)))
+
+ if errors:
+ raise shutil.Error(errors)
+ return dst
+
def update_notebooks_params(working_dir: Path, new_values):
"""Modify parameters in copied notebooks"""
for nb_file in working_dir.glob('*.ipynb'):
@@ -22,21 +69,50 @@ def update_notebooks_params(working_dir: Path, new_values):
new_nb = replace_definitions(nb, params, execute=False, lang='python')
nbformat.write(new_nb, nb_file)
-def new_report_path(old_out_folder, old_report_path, new_out_folder):
- """Update report path if it was in the old output folder"""
- try:
- report_in_output = Path(old_report_path).relative_to(old_out_folder)
- except ValueError:
- return old_report_path
- else:
- return str(Path(new_out_folder, report_in_output))
+
+def get_python(args, py_version):
+ if args.env_cache:
+ reqs = (args.from_dir / 'requirements.txt').read_text('utf-8')
+ reqs = munge_requirements(reqs)
+ env_mgr = EnvsManager(
+ Path(args.env_cache), PyenvEnvMaker()
+ )
+ return env_mgr.get_env(py_version, reqs).resolve() / 'bin' / 'python'
+ elif args.python:
+ return args.python.resolve()
+ return Path(sys.executable)
+
+
+def munge_requirements(reqs: str):
+ # Ugly workaround - these internal packages are dependencies of
+ # pycalibration, but 'pip freeze' shows them differently to how they are
+ # specified in setup.py, which causes conflicts in 'pip install'. For now,
+ # so we can recalibrate old data, we'll cut them out of the requirements
+ # list and rely on pycalibration specifying them as pinned dependencies.
+ reqs = reqs.splitlines(keepends=False)
+ reqs = [r for r in reqs if not r.startswith(
+ ('iCalibrationDB', 'XFELDetectorAnalysis')
+ )]
+ return '\n'.join(reqs)
+
def main(argv=None):
ap = argparse.ArgumentParser()
ap.add_argument("from_dir", type=Path, help="A directory containing steps.json")
- ap.add_argument("--python", help="Path to Python executable to run notebooks")
+
+ env_args = ap.add_mutually_exclusive_group()
+ env_args.add_argument("--python", type=Path,
+ help="Path to Python executable to run notebooks")
+ env_args.add_argument(
+ "--env-cache",
+ help="Make/reuse a virtualenv in this cache directory with the Python "
+ "version & library versions these notebooks previously used. "
+ "This requires the pyenv command to be available."
+ )
ap.add_argument("--out-folder", help="Directory to put output data")
+ ap.add_argument("--report-to", help="Location to save PDF report")
ap.add_argument("--slurm-partition", help="Submit jobs in this Slurm partition")
+ ap.add_argument("--slurm-mem", type=int, help="Requested node RAM in GB")
ap.add_argument('--no-cluster-job', action="store_true",
help="Run notebooks here, not in cluster jobs")
args = ap.parse_args(argv)
@@ -47,31 +123,37 @@ def main(argv=None):
start_time = datetime.now()
run_uuid = f"t{start_time:%y%m%d_%H%M%S}"
- working_dir = Path(temp_path, f'slurm_out_repeat_{run_uuid}')
- shutil.copytree(
- args.from_dir, working_dir, ignore=shutil.ignore_patterns('slurm-*.out')
+ cal_work_dir = Path(temp_path, f'slurm_out_repeat_{run_uuid}')
+ copytree_no_metadata(
+ args.from_dir, cal_work_dir, ignore=shutil.ignore_patterns('slurm-*.out')
)
- print(f"New working directory: {working_dir}")
+ print(f"New working directory: {cal_work_dir}")
- cal_metadata = CalibrationMetadata(working_dir)
- prev_parameters = cal_metadata['calibration-configurations']
+ cal_metadata = CalibrationMetadata(cal_work_dir)
+ parameters = cal_metadata['calibration-configurations']
- report_path = cal_metadata['report-path']
- out_folder = prev_parameters['out-folder']
+ out_folder = parameters['out-folder']
params_to_set = {'metadata_folder': "."}
if args.out_folder:
- report_path = new_report_path(out_folder, report_path, args.out_folder)
- out_folder = prev_parameters['out-folder'] = args.out_folder
+ out_folder = parameters['out-folder'] = args.out_folder
params_to_set['out_folder'] = out_folder
- cal_metadata.save()
- update_notebooks_params(working_dir, params_to_set)
+ update_notebooks_params(cal_work_dir, params_to_set)
+
+ if args.report_to:
+ report_to = args.report_to
+ else: # Default to saving report in output folder
+ report_to = str(Path(out_folder, f'xfel-calibrate-repeat-{run_uuid}'))
+ cal_metadata['report-path'] = f'{report_to}.pdf'
+
+ cal_metadata.save()
# finalize & some notebooks expect yaml metadata in the output folder
Path(out_folder).mkdir(parents=True, exist_ok=True)
- shutil.copy(working_dir / 'calibration_metadata.yml', out_folder)
+ shutil.copy(cal_work_dir / 'calibration_metadata.yml', out_folder)
+ py_version = cal_metadata.get('python-environment', {}).get('python-version')
job_chain = JobChain.from_dir(
- working_dir, python=(args.python or sys.executable)
+ cal_work_dir, python=get_python(args, py_version)
)
if args.no_cluster_job:
job_chain.run_direct()
@@ -79,20 +161,21 @@ def main(argv=None):
else:
joblist = job_chain.submit_jobs(SlurmOptions(
partition=args.slurm_partition,
+ mem=args.slurm_mem,
))
- fmt_args = {'run_path': working_dir,
+ fmt_args = {'run_path': cal_work_dir,
'out_path': out_folder,
'version': get_pycalib_version(),
- 'report_to': report_path,
- 'in_folder': prev_parameters['in-folder'],
+ 'report_to': report_to,
+ 'in_folder': parameters['in-folder'],
'request_time': start_time.strftime('%Y-%m-%dT%H:%M:%S'),
'submission_time': start_time.strftime('%Y-%m-%dT%H:%M:%S'),
}
joblist.append(run_finalize(
fmt_args=fmt_args,
- temp_path=working_dir,
+ cal_work_dir=cal_work_dir,
job_list=joblist,
sequential=args.no_cluster_job,
partition=args.slurm_partition
diff --git a/tests/conftest.py b/tests/conftest.py
index fd71bd602e4b7921a05c95639f2ddce0c797f558..e5a45a293cfd107829309ce18e5f10c914a05107 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,10 +1,40 @@
import socket
from functools import lru_cache
from pathlib import Path
+from tempfile import TemporaryDirectory
+import extra_data.tests.make_examples as make_examples
import pytest
+@pytest.fixture(scope='session')
+def mock_agipd1m_run():
+ with TemporaryDirectory() as td:
+ make_examples.make_agipd1m_run(td)
+ yield td
+
+
+@pytest.fixture(scope='session')
+def mock_agipd1m_run_old():
+ # No gain.value or bunchStructure.repetitionRate in /CONTROL/
+ with TemporaryDirectory() as td:
+ make_examples.make_agipd1m_run(
+ td, rep_rate=False,
+ gain_setting=False,
+ integration_time=False,
+ bias_voltage=False
+ )
+ yield td
+
+
+@pytest.fixture(scope='session')
+def mock_agipd500k_run():
+ # No gain.value or bunchStructure.repetitionRate in /CONTROL/
+ with TemporaryDirectory() as td:
+ make_examples.make_agipd500k_run(td)
+ yield td
+
+
def pytest_addoption(parser):
parser.addoption(
"--no-gpfs",
diff --git a/tests/test_agipdlib.py b/tests/test_agipdlib.py
new file mode 100644
index 0000000000000000000000000000000000000000..ad1a92f42f956d291adebedbfb6df61a929d4e32
--- /dev/null
+++ b/tests/test_agipdlib.py
@@ -0,0 +1,200 @@
+from datetime import datetime
+
+from extra_data import RunDirectory
+
+from cal_tools.agipdlib import AgipdCtrl
+
+SPB_AGIPD_INST_SRC = 'SPB_DET_AGIPD1M-1/DET/0CH0:xtdf'
+CTRL_SRC = 'SPB_IRU_AGIPD1M1/MDL/FPGA_COMP'
+
+
+def test_get_acq_rate_ctrl(mock_agipd1m_run):
+ # Current up to date data with acq_rate stored
+ # as repetition rate in slow data.
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=CTRL_SRC)
+
+ acq_rate = agipd_ctrl._get_acq_rate_ctrl()
+ assert isinstance(acq_rate, float)
+ assert acq_rate == 4.5
+
+
+def test_get_acq_rate_instr(mock_agipd1m_run_old):
+
+ # Old data with no stored acq_rate in slow data.
+ # Reading acq_rate from fast data.
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run_old),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=None)
+ acq_rate = agipd_ctrl._get_acq_rate_instr()
+ assert isinstance(acq_rate, float)
+ assert acq_rate == 4.5
+
+
+def test_get_acq_rate(mock_agipd1m_run, mock_agipd1m_run_old):
+
+ # Current up to date data with acq_rate stored
+ # as repetition rate in slow data.
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=CTRL_SRC)
+
+ acq_rate = agipd_ctrl.get_acq_rate()
+ assert isinstance(acq_rate, float)
+ assert acq_rate == 4.5
+
+ # Old data with no stored acq_rate in slow data.
+ # Reading acq_rate from fast data.
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run_old),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=None)
+ acq_rate = agipd_ctrl.get_acq_rate()
+ assert isinstance(acq_rate, float)
+ assert acq_rate == 4.5
+
+
+def test_get_num_cells(mock_agipd1m_run):
+
+ # Reading number of memory cells from fast data.
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=None)
+ mem_cells = agipd_ctrl.get_num_cells()
+ assert isinstance(mem_cells, int)
+ assert mem_cells == 64
+
+
+def test_get_gain_setting_ctrl(mock_agipd1m_run):
+ # Reading gain setting from slow data.
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=CTRL_SRC)
+ gain_setting = agipd_ctrl._get_gain_setting_ctrl()
+ assert isinstance(gain_setting, int)
+ assert gain_setting == 0
+
+
+def test_get_gain_setting_ctrl_old(mock_agipd1m_run_old):
+
+ # Reading gain setting from setupr and .....
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run_old),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=CTRL_SRC)
+ gain_setting = agipd_ctrl._get_gain_setting_ctrl_old()
+ assert isinstance(gain_setting, int)
+ assert gain_setting == 0
+
+
+def test_get_gain_setting(mock_agipd1m_run, mock_agipd1m_run_old):
+
+ # Reading gain setting from slow data.
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=CTRL_SRC)
+ gain_setting = agipd_ctrl.get_gain_setting()
+ assert isinstance(gain_setting, int)
+ assert gain_setting == 0
+
+ # Reading gain setting from setupr and patternTypeIndex
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run_old),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=CTRL_SRC)
+ gain_setting = agipd_ctrl.get_gain_setting()
+ assert isinstance(gain_setting, int)
+ assert gain_setting == 0
+
+ # Old data without gain setting.
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run_old),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=CTRL_SRC)
+
+ # Pass old creation_date before introducing gain_setting in 2020.
+ gain_setting = agipd_ctrl.get_gain_setting(
+ creation_time=datetime.strptime(
+ "2019-12-16T08:52:25",
+ "%Y-%m-%dT%H:%M:%S"))
+
+ assert gain_setting is None
+
+
+def test_get_bias_voltage(
+ mock_agipd1m_run,
+ mock_agipd500k_run,
+ mock_agipd1m_run_old
+):
+ # Read bias voltage for HED_DET_AGIPD500K from slow data.
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd500k_run),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=CTRL_SRC)
+ bias_voltage = agipd_ctrl.get_bias_voltage(
+ karabo_id_control="HED_EXP_AGIPD500K2G")
+
+ assert isinstance(bias_voltage, int)
+ assert bias_voltage == 200
+
+ # Read bias voltage for SPB_DET_AGIPD1M-1 and MID_DET_AGIPD1M-1
+ # from slow data.
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=CTRL_SRC)
+ bias_voltage = agipd_ctrl.get_bias_voltage(
+ karabo_id_control="SPB_IRU_AGIPD1M1")
+ assert isinstance(bias_voltage, int)
+ assert bias_voltage == 300
+
+ # Fail to read bias voltage for SPB_DET_AGIPD1M-1 and MID_DET_AGIPD1M-1
+ # from old data and return default value.
+ # TODO: Uncomment this code when the corresponding fix
+ # in Extra-data is released.
+ # agipd_ctrl = AgipdCtrl(
+ # run_dc=RunDirectory(mock_agipd1m_run_old),
+ # image_src=SPB_AGIPD_INST_SRC,
+ # ctrl_src=CTRL_SRC)
+ # bias_voltage = agipd_ctrl.get_bias_voltage(
+ # karabo_id_control="SPB_IRU_AGIPD1M1")
+ # assert isinstance(bias_voltage, int)
+ # assert bias_voltage == 300
+
+
+def test_get_integration_time(mock_agipd1m_run, mock_agipd1m_run_old):
+ # Read integration time.
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=CTRL_SRC)
+ integration_time = agipd_ctrl.get_integration_time()
+ assert isinstance(integration_time, int)
+ assert integration_time == 15
+
+ # Integration time is not available in slow data.
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run_old),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=CTRL_SRC)
+ integration_time = agipd_ctrl.get_integration_time()
+ assert isinstance(integration_time, int)
+ assert integration_time == 12
+
+
+def test_get_gain_mode(mock_agipd1m_run):
+ # Read gain mode from slow data.
+ agipd_ctrl = AgipdCtrl(
+ run_dc=RunDirectory(mock_agipd1m_run),
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=CTRL_SRC)
+ gain_mode = agipd_ctrl.get_gain_mode()
+ assert isinstance(gain_mode, int)
+ assert gain_mode == 0
diff --git a/tests/test_agipdutils.py b/tests/test_agipdutils.py
new file mode 100644
index 0000000000000000000000000000000000000000..7c0f4f28755ecbda89b109ef83bf801da80d97d5
--- /dev/null
+++ b/tests/test_agipdutils.py
@@ -0,0 +1,39 @@
+
+import pytest
+import numpy as np
+
+from cal_tools.agipdutils import cast_array_inplace
+
+
+@pytest.mark.parametrize(
+ 'dtype_str', ['f8', 'f4', 'f2', 'i4', 'i2', 'i1', 'u4', 'u2', 'u1'])
+def test_downcast_array_inplace(dtype_str):
+ """Test downcasting an array in-place."""
+
+ dtype = np.dtype(dtype_str)
+
+ ref_data = (np.random.rand(2, 3, 4) * 100)
+ orig_data = ref_data.copy()
+ cast_data = cast_array_inplace(orig_data, dtype)
+
+ np.testing.assert_allclose(cast_data, ref_data.astype(dtype))
+ assert np.may_share_memory(orig_data, cast_data)
+ assert cast_data.dtype == dtype
+ assert cast_data.flags.c_contiguous
+ assert cast_data.flags.aligned
+ assert not cast_data.flags.owndata
+
+
+def test_upcast_array_inplace():
+ """Test whether upcasting an array in-place fails."""
+
+ with pytest.raises(Exception):
+ cast_array_inplace(
+ np.random.rand(4, 5, 6).astype(np.float32), np.float64)
+
+
+def test_noncontiguous_cast_inplace():
+ """Test whether casting a non-contiguous array in-place fails."""
+
+ with pytest.raises(Exception):
+ cast_array_inplace(np.random.rand(4, 5, 6).T, np.int32)
diff --git a/tests/test_cal_tools.py b/tests/test_cal_tools.py
index 6e7f44b279dbf59788acf1e1f73fd0e21e1cd7e9..ad493e46c1bf1cd15336508c9dbc98f296ae80af 100644
--- a/tests/test_cal_tools.py
+++ b/tests/test_cal_tools.py
@@ -12,11 +12,11 @@ from cal_tools.agipdlib import AgipdCorrections, CellRange
from cal_tools.plotting import show_processed_modules
from cal_tools.tools import (
creation_date_file_metadata,
- creation_date_metadata_client,
creation_date_train_timestamp,
get_dir_creation_date,
get_from_db,
get_pdu_from_db,
+ map_seq_files,
module_index_to_qm,
send_to_db,
)
@@ -37,6 +37,7 @@ WRONG_CAL_DB_INTERFACE = "tcp://max-exfl017:0000"
PROPOSAL = 900113
+
@pytest.fixture
def _agipd_const_cond():
# AGIPD dark offset metadata
@@ -60,6 +61,30 @@ def test_show_processed_modules():
assert 'LDP' in err.value()
+@pytest.mark.requires_gpfs
+@pytest.mark.parametrize(
+ "karabo_da,sequences,expected_len",
+ [
+ ("AGIPD00", [-1], 3),
+ ("AGIPD00", [0, 1], 2),
+ ("EPIX01", [-1], 0),
+ ("AGIPD00", [117], 0),
+ ],
+)
+def test_map_seq_files(karabo_da, sequences, expected_len):
+ run_folder = Path('/gpfs/exfel/exp/CALLAB/202031/p900113/raw/r9983')
+ expected_dict = {karabo_da: []}
+
+ if expected_len:
+ sequences = range(expected_len) if sequences == [-1] else sequences
+ expected_dict = {
+ karabo_da: [run_folder / f"RAW-R9983-AGIPD00-S0000{s}.h5" for s in sequences] # noqa
+ }
+
+ assert map_seq_files(run_folder, [karabo_da], sequences) == (
+ expected_dict, expected_len)
+
+
@pytest.mark.requires_gpfs
def test_dir_creation_date():
"""This test is based on not connecting to MDC and failing to use
@@ -86,23 +111,17 @@ def test_raise_dir_creation_date():
assert e.value.args[1] == Path(folder) / 'r0004'
-@pytest.mark.requires_mdc
-def test_creation_date_metadata_client():
-
- date = creation_date_metadata_client(PROPOSAL, 9983)
- assert isinstance(date, datetime)
- assert str(date) == '2020-09-23 13:30:00+00:00'
-
-
@pytest.mark.requires_gpfs
def test_creation_date_file_metadata():
- date = creation_date_file_metadata(open_run(PROPOSAL, 9983))
+ date = creation_date_file_metadata(
+ Path('/gpfs/exfel/exp/CALLAB/202031/p900113/raw/r9983'))
assert isinstance(date, datetime)
assert str(date) == '2020-09-23 13:30:50+00:00'
# Old run without METADATA/CreationDate
- date = creation_date_file_metadata(open_run(PROPOSAL, 9999))
+ date = creation_date_file_metadata(
+ Path('/gpfs/exfel/exp/CALLAB/202031/p900113/raw/r9999'))
assert date is None
diff --git a/tests/test_agipdalgs.py b/tests/test_cythonalgs.py
similarity index 52%
rename from tests/test_agipdalgs.py
rename to tests/test_cythonalgs.py
index 6296eb0494d656596131c8ee65fb52dafe1638bf..cff7fe33b8da91059309d0c05d64673986416bba 100644
--- a/tests/test_agipdalgs.py
+++ b/tests/test_cythonalgs.py
@@ -1,2 +1,3 @@
def test_import():
from cal_tools import agipdalgs # noqa
+ from cal_tools.gotthard2 import gotthard2algs # noqa
diff --git a/tests/test_gotthard2algs.py b/tests/test_gotthard2algs.py
new file mode 100644
index 0000000000000000000000000000000000000000..6b5fbbd63116b25352574b3432a2451b85a4c8cc
--- /dev/null
+++ b/tests/test_gotthard2algs.py
@@ -0,0 +1,64 @@
+import numpy as np
+import pytest
+
+from cal_tools.gotthard2.gotthard2algs import convert_to_10bit, correct_train
+
+
+def test_convert_to_10bit():
+ """Test converting 12bit Gotthard2 image to 10bit."""
+ n_stripes = 10
+ n_pulses = 500
+
+ # Define LUT, raw data 12 bit, and raw data 10bit array.
+ lut = np.array(
+ [[list(range(4096//2))*2, list(range(4096//2, 4096))*2]] * n_stripes,
+ dtype=np.uint16
+ )
+ raw_data = np.array([list(range(n_stripes))]*n_pulses, dtype=np.uint16)
+ raw_data10bit = np.zeros(raw_data.shape, dtype=np.float32)
+
+ result = np.concatenate(
+ [
+ np.array(x)[:, None] for x in [
+ list(range(n_stripes)),
+ list(range(2048, 2048+n_stripes))
+ ] * (n_pulses//2)], axis=1, dtype=np.float32,
+ ).T
+
+ convert_to_10bit(raw_data, lut.astype(np.uint16), raw_data10bit)
+ assert np.allclose(result, raw_data10bit)
+
+
+@pytest.mark.parametrize("gain_corr", [True, False])
+def test_correct_train(gain_corr):
+ """Test gotthard2 correction function."""
+
+ raw_d = np.random.randn(2700, 1280).astype(np.float32)
+ gain = np.random.choice([0, 1, 2], size=(2700, 1280)).astype(np.uint8)
+
+ offset = np.random.randn(1280, 2, 3).astype(np.float32)
+ gain_map = np.random.randn(1280, 2, 3).astype(np.float32)
+ badpixles = np.zeros_like(offset).astype(np.uint32).astype(np.uint32)
+
+ test_data = raw_d.copy()
+ mask = np.zeros_like(test_data).astype(np.uint32)
+
+ correct_train(
+ data=test_data, mask=mask, gain=gain, offset_map=offset,
+ gain_map=gain_map, bpix_map=badpixles, apply_gain=gain_corr,
+ )
+
+ ref_data = raw_d.copy()
+
+ ref_data[::2, :] -= np.choose(
+ gain[::2, :], np.moveaxis(offset[:, 0, :], 1, 0))
+ ref_data[1::2, :] -= np.choose(
+ gain[1::2, :], np.moveaxis(offset[:, 1, :], 1, 0))
+
+ if gain_corr:
+ ref_data[::2, :] /= np.choose(
+ gain[::2, :], np.moveaxis(gain_map[:, 0, :], 1, 0))
+ ref_data[1::2, :] /= np.choose(
+ gain[1::2, :], np.moveaxis(gain_map[:, 1, :], 1, 0))
+
+ assert np.allclose(test_data, test_data)
diff --git a/tests/test_update_config.py b/tests/test_update_config.py
index e06bc052448d35800380f3a80f9280fe23ba8b4a..7a4c79965051b5a31847ca9c02875fd08eaf1252 100644
--- a/tests/test_update_config.py
+++ b/tests/test_update_config.py
@@ -84,27 +84,30 @@ def test_main(capsys):
EXPECTED_CONF = [
{
- 'force-hg-if-below': {'typ': int},
- 'rel-gain': {'typ': bool},
- 'xray-gain': {'typ': bool},
- 'blc-noise': {'typ': bool},
- 'blc-set-min': {'typ': bool},
- 'dont-zero-nans': {'typ': bool},
- 'dont-zero-orange': {'typ': bool},
- 'max-pulses': {'typ': list,
+ 'force-hg-if-below': {'type': int},
+ 'rel-gain': {'type': bool},
+ 'xray-gain': {'type': bool},
+ 'blc-noise': {'type': bool},
+ 'blc-set-min': {'type': bool},
+ 'dont-zero-nans': {'type': bool},
+ 'dont-zero-orange': {'type': bool},
+ 'max-pulses': {'type': list,
'msg': 'Range list of maximum pulse indices '
'(--max-pulses start end step). '
'3 max input elements. '},
- 'no-rel-gain': {'typ': bool},
- 'no-xray-gain': {'typ': bool},
- 'no-blc-noise': {'typ': bool},
- 'no-blc-set-min': {'typ': bool},
- 'no-dont-zero-nans': {'typ': bool},
- 'no-dont-zero-orange': {'typ': bool}
+ 'use-litframe-finder': {'type': str},
+ 'litframe-device-id': {'type': str},
+ 'energy-threshold': {'type': int},
+ 'no-rel-gain': {'type': bool},
+ 'no-xray-gain': {'type': bool},
+ 'no-blc-noise': {'type': bool},
+ 'no-blc-set-min': {'type': bool},
+ 'no-dont-zero-nans': {'type': bool},
+ 'no-dont-zero-orange': {'type': bool}
},
{
'karabo-da': {
- 'typ': list,
+ 'type': list,
'choices': [
'AGIPD00', 'AGIPD01', 'AGIPD02', 'AGIPD03',
'AGIPD04', 'AGIPD05', 'AGIPD06', 'AGIPD07',
diff --git a/tests/test_webservice.py b/tests/test_webservice.py
index a61c7e12be7ac6515b7105fc1ee9699449f39afd..8ce9afe6c0ee756e026fcbf92afe0ae6f265e5d5 100644
--- a/tests/test_webservice.py
+++ b/tests/test_webservice.py
@@ -1,5 +1,7 @@
+from collections import namedtuple
import logging
import sys
+import datetime as dt
from pathlib import Path
from unittest import mock
@@ -15,9 +17,25 @@ from webservice.webservice import ( # noqa: import not at top of file
run_action,
wait_on_transfer,
get_slurm_partition,
- get_slurm_nice
+ get_slurm_nice,
+ config,
)
+VALID_BEAMTIME = {
+ "begin_at": (dt.datetime.today() - dt.timedelta(days=1)).isoformat(),
+ "description": None,
+ "end_at": (dt.datetime.today() + dt.timedelta(days=1)).isoformat(),
+ "flg_available": True,
+ "id": 772,
+ }
+
+INVALID_BEAMTIME = {
+ "begin_at": "1818-05-05T00:00:00.000+02:00",
+ "description": "Karl",
+ "end_at": "1883-03-14T00:00:00.000+02:00",
+ "flg_available": True,
+ "id": 773,
+}
@pytest.mark.requires_gpfs
def test_check_files():
@@ -240,22 +258,33 @@ async def test_run_action(mode, cmd, retcode, expected, monkeypatch):
@pytest.mark.asyncio
@pytest.mark.parametrize(
- 'proposal_number, action, mock_proposal_status, expected_result',
+ 'proposal_number, action, mock_proposal_status, mock_beamtimes, expected_result',
[
- (42, 'correct', 'A', 'upex-middle'), # active
- ('42', 'dark', 'R', 'upex-high'), # active
- (404, 'correct', 'FR', 'exfel'), # finished and reviewed
- (404, 'dark', 'CLO', 'exfel'), # closed
+ (42, 'correct', 'A', [INVALID_BEAMTIME, VALID_BEAMTIME], 'upex-middle'), # active
+ ('42', 'dark', 'R', [INVALID_BEAMTIME, VALID_BEAMTIME], 'upex-high'), # active
+ (404, 'correct', 'FR', [INVALID_BEAMTIME, VALID_BEAMTIME], 'exfel'), # finished and reviewed
+ (404, 'dark', 'CLO', [INVALID_BEAMTIME, VALID_BEAMTIME], 'exfel'), # closed
+ (42, 'correct', 'A', [INVALID_BEAMTIME, INVALID_BEAMTIME], 'exfel'), # active
+ (42, 'correct', 'A', [INVALID_BEAMTIME], 'exfel'), # active but not in beamtime
+ (42, 'correct', 'A', [VALID_BEAMTIME], 'upex-middle'), # active
+ ('42', 'dark', 'R', [VALID_BEAMTIME, {}], 'upex-high'), # active
+ (42, 'correct', 'A', [], 'exfel'), # active, no beamtime?
+ (42, 'correct', 'A', None, 'exfel'), # active, no beamtime?
],
)
-async def test_get_slurm_partition(proposal_number,
- action,
- mock_proposal_status,
- expected_result):
+async def test_get_slurm_partition(
+ proposal_number, action, mock_proposal_status, mock_beamtimes, expected_result
+):
+
+ return_value = {
+ "flg_beamtime_status": mock_proposal_status,
+ }
+ if mock_beamtimes is not None: # Test that we handle missing field
+ return_value["beamtimes"] = mock_beamtimes
response = mock.Mock()
response.status_code = 200
- response.json = mock.Mock(return_value={'flg_beamtime_status': mock_proposal_status})
+ response.json = mock.Mock(return_value=return_value)
client = mock.Mock()
client.get_proposal_by_number_api = mock.Mock(
return_value=response)
@@ -263,6 +292,52 @@ async def test_get_slurm_partition(proposal_number,
ret = await get_slurm_partition(client, action, proposal_number)
assert ret == expected_result
+
+@pytest.mark.parametrize(
+ 'commissioning, run_age_days, expected_partition',
+ [
+ (False, 0, "upex-middle"),
+ (True, 0, "upex-middle"),
+ (False, config["correct"]["commissioning-max-age-days"], "upex-middle"),
+ (True, config["correct"]["commissioning-max-age-days"], "upex-middle"),
+ (False, config["correct"]["commissioning-max-age-days"]+1, "upex-middle"),
+ (True, config["correct"]["commissioning-max-age-days"]+1, "exfel"),
+ ]
+)
+@pytest.mark.asyncio
+async def test_get_slurm_partition_run_age(
+ commissioning: bool,
+ run_age_days: int,
+ expected_partition: str,
+):
+ # NOTE: non-zero at cycle index 4 (`str(cycle)[4]`) indicates commissioning
+ cycle = 202231 if commissioning else 202201
+
+ now = dt.datetime.now().astimezone()
+ run_dt = now - dt.timedelta(days=run_age_days)
+ run_str = run_dt.strftime("%Y-%m-%dT%H:%M:%S.%f%z")
+
+ # Note that, as the fields are independent, this mocks both the
+ # response for `get_proposal_by_number_api` AND `get_run_by_id_api`
+ response = mock.Mock()
+ response.status_code = 200
+ response.json = lambda: {
+ "flg_beamtime_status": "A",
+ "begin_at": run_str,
+ "beamtimes": [INVALID_BEAMTIME, VALID_BEAMTIME],
+ }
+
+ client = mock.Mock()
+ client.get_proposal_by_number_api = mock.Mock(return_value=response)
+ client.get_run_by_id_api = mock.Mock(return_value=response)
+
+ ret = await get_slurm_partition(
+ client, "correct", 1, run_id=1, cycle=cycle
+ )
+
+ assert ret == expected_partition
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
'cycle, num_jobs, expected_result',
@@ -303,3 +378,4 @@ async def test_get_slurm_nice_fails(fp):
returncode=0)
assert await get_slurm_nice('exfel', 'SPB', '202201') == 0
+
diff --git a/tests/test_xfel_calibrate/conftest.py b/tests/test_xfel_calibrate/conftest.py
index 7533d2f278397fc234f2c780aebdfc217622ade2..a98c410259385d6dc4a7d837b374331681635996 100644
--- a/tests/test_xfel_calibrate/conftest.py
+++ b/tests/test_xfel_calibrate/conftest.py
@@ -169,7 +169,7 @@ class CalibrateCall:
def __init__(
self,
- tmp_path,
+ reports_dir,
capsys,
*,
detector: str,
@@ -211,12 +211,11 @@ class CalibrateCall:
extra_args (List[str], optional): Additional arguments to pass to the
command. Defaults to None.
"""
- self.tmp_path = tmp_path
+ self.reports_dir = reports_dir
self.in_folder = in_folder
self.out_folder = out_folder
- self.args = []
- self.args.extend([command, detector, cal_type])
+ self.args = [command, detector, cal_type, '--report-to', str(reports_dir)]
if in_folder:
self.args.extend(["--in-folder", str(self.in_folder)])
if out_folder:
@@ -225,13 +224,12 @@ class CalibrateCall:
if extra_args:
self.args.extend(extra_args)
- with mock.patch.object(calibrate, "temp_path", str(tmp_path)):
- calibrate.run(argv=self.args)
+ calibrate.run(argv=self.args)
- out, err = capsys.readouterr()
+ out, err = capsys.readouterr()
- self.out: str = out
- self.err: str = err
+ self.out: str = out
+ self.err: str = err
Paths = NamedTuple(
"Paths",
@@ -244,8 +242,8 @@ class CalibrateCall:
)
self.paths = Paths(
- notebooks=list(self.tmp_path.glob("**/*/*.ipynb")),
- run_calibrate=list(self.tmp_path.glob("**/*/run_calibrate.sh"))[0],
- finalize=list(self.tmp_path.glob("**/*/finalize.py"))[0],
- InputParameters=list(self.tmp_path.glob("**/*/InputParameters.rst"))[0],
+ notebooks=list(self.reports_dir.glob("**/*/*.ipynb")),
+ run_calibrate=list(self.reports_dir.glob("**/*/run_calibrate.sh"))[0],
+ finalize=list(self.reports_dir.glob("**/*/finalize.py"))[0],
+ InputParameters=list(self.reports_dir.glob("**/*/InputParameters.rst"))[0],
)
diff --git a/tests/test_xfel_calibrate/test_cli.py b/tests/test_xfel_calibrate/test_cli.py
index 1bb0c1a2794bfde1932f4b4702e63b18facdfa79..097e0f3c27d18b3f96fca180b5050b5c73ba4bf7 100644
--- a/tests/test_xfel_calibrate/test_cli.py
+++ b/tests/test_xfel_calibrate/test_cli.py
@@ -152,7 +152,7 @@ class TestTutorialNotebook:
):
assert "sbatch" in calibrate_call.out
assert "--job-name xfel_calibrate" in calibrate_call.out
- assert str(calibrate_call.tmp_path) in calibrate_call.out
+ assert str(calibrate_call.reports_dir) in calibrate_call.out
assert calibrate_call.err == ""
@@ -210,11 +210,11 @@ class TestTutorialNotebook:
expected_contains = {
"request_time": str(today),
"submission_time": str(today),
- "run_path": str(calibrate_call.tmp_path),
+ "cal_work_dir": str(calibrate_call.reports_dir),
# TODO: add a test checking that the out folder is correct
# reffer to: https://git.xfel.eu/gitlab/detectors/pycalibration/issues/52
"out_path": str(mock_proposal.path_proc),
- "report_to": str(mock_proposal.path_proc),
+ "report_to": str(calibrate_call.reports_dir),
}
# Pull the keyword arguments out of the finalize function call via the AST,
diff --git a/webservice/config/serve_overview.yaml b/webservice/config/serve_overview.yaml
index 10dea021628f61ba79facd76e7f44d63dbbc7f78..6dfc314c11787f5aae84eb3e97b9a0d9364900d0 100644
--- a/webservice/config/serve_overview.yaml
+++ b/webservice/config/serve_overview.yaml
@@ -11,7 +11,6 @@ templates:
shell-commands:
total-jobs: "sinfo -p exfel -o %A --noheader"
tail-log: "tail -5000 web.log"
- cat-log: "cat web.log"
tail-log-monitor: "tail -5000 monitor.log"
run-candidates:
diff --git a/webservice/config/webservice.yaml b/webservice/config/webservice.yaml
index c91b4b923f70df2982f996d0206d367c0a70c37e..f57b6431f3c78aebb539f4b9727a458985b4eaa3 100644
--- a/webservice/config/webservice.yaml
+++ b/webservice/config/webservice.yaml
@@ -1,60 +1,63 @@
+---
config-repo:
- url: "@note add this to secrets file"
- local-path: "@format {env[HOME]}/calibration_config"
+ url: "@note add this to secrets file"
+ local-path: "@format {env[HOME]}/calibration_config"
web-service:
- port: 5555
- bind-to: tcp://*
- allowed-ips:
- job-db: "@format {this.webservice_dir}/webservice_jobs.sqlite"
- job-update-interval: 60
- job-timeout: 3600
+ port: 5555
+ bind-to: tcp://*
+ allowed-ips:
+ job-db: "@format {this.webservice_dir}/webservice_jobs.sqlite"
+ job-update-interval: 60
+ job-timeout: 3600
metadata-client:
- user-id: "@note add this to secrets file"
- user-secret: "@note add this to secrets file"
- user-email: "@note add this to secrets file"
- metadata-web-app-url: "https://in.xfel.eu/metadata"
- token-url: "https://in.xfel.eu/metadata/oauth/token"
- refresh-url: "https://in.xfel.eu/metadata/oauth/token"
- auth-url: "https://in.xfel.eu/metadata/oauth/authorize"
- scope: ""
- base-api-url: "https://in.xfel.eu/metadata/api/"
+ user-id: "@note add this to secrets file"
+ user-secret: "@note add this to secrets file"
+ user-email: "@note add this to secrets file"
+ metadata-web-app-url: "https://in.xfel.eu/metadata"
+ token-url: "https://in.xfel.eu/metadata/oauth/token"
+ refresh-url: "https://in.xfel.eu/metadata/oauth/token"
+ auth-url: "https://in.xfel.eu/metadata/oauth/authorize"
+ scope: ""
+ base-api-url: "https://in.xfel.eu/metadata/api/"
kafka:
- brokers:
+ brokers:
- it-kafka-broker01.desy.de
- it-kafka-broker02.desy.de
- it-kafka-broker03.desy.de
- topic: xfel-test-offline-cal
+ topic: xfel-test-offline-cal
correct:
- in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
- out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}/{run}
- commissioning-penalty: 1250
- job-penalty: 2
- cmd : >-
- python -m xfel_calibrate.calibrate {detector} CORRECT
- --slurm-scheduling {sched_prio}
- --slurm-partition {partition}
- --request-time {request_time}
- --slurm-name {action}_{instrument}_{detector}_{cycle}_p{proposal}_{runs}
- --report-to /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/usr/Reports/{runs}/{det_instance}_{action}_{proposal}_{runs}_{time_stamp}
- --cal-db-timeout 300000
- --cal-db-interface tcp://max-exfl016:8015#8044
+ in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
+ out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}/{run}
+ reports-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/usr/Reports/{runs}/
+ commissioning-penalty: 1250
+ commissioning-max-age-days: 3
+ job-penalty: 2
+ cmd: >-
+ python -m xfel_calibrate.calibrate {detector} CORRECT
+ --slurm-scheduling {sched_prio}
+ --slurm-partition {partition}
+ --request-time {request_time}
+ --slurm-name {action}_{instrument}_{detector}_{cycle}_p{proposal}_{runs}
+ --report-to /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/usr/Reports/{runs}/{det_instance}_{action}_{proposal}_{runs}_{time_stamp}
+ --cal-db-timeout 300000
+ --cal-db-interface tcp://max-exfl016:8015#8044
dark:
- in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
- out-folder: /gpfs/exfel/u/usr/{instrument}/{cycle}/p{proposal}/dark/runs_{runs}
- commissioning-penalty: 1250
- job-penalty: 2
- cmd: >-
- python -m xfel_calibrate.calibrate {detector} DARK
- --concurrency-par karabo_da
- --slurm-scheduling {sched_prio}
- --slurm-partition {partition}
- --request-time {request_time}
- --slurm-name {action}_{instrument}_{detector}_{cycle}_p{proposal}_{runs}
- --report-to /gpfs/exfel/d/cal/caldb_store/xfel/reports/{instrument}/{det_instance}/{action}/{action}_{proposal}_{runs}_{time_stamp}
- --cal-db-interface tcp://max-exfl016:8015#8044
- --db-output
+ in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
+ out-folder: /gpfs/exfel/u/usr/{instrument}/{cycle}/p{proposal}/dark/runs_{runs}
+ commissioning-penalty: 1250
+ job-penalty: 2
+ cmd: >-
+ python -m xfel_calibrate.calibrate {detector} DARK
+ --concurrency-par karabo_da
+ --slurm-scheduling {sched_prio}
+ --slurm-partition {partition}
+ --request-time {request_time}
+ --slurm-name {action}_{instrument}_{detector}_{cycle}_p{proposal}_{runs}
+ --report-to /gpfs/exfel/d/cal/caldb_store/xfel/reports/{instrument}/{det_instance}/{action}/{action}_{proposal}_{runs}_{time_stamp}
+ --cal-db-interface tcp://max-exfl016:8015#8044
+ --db-output
diff --git a/webservice/job_monitor.py b/webservice/job_monitor.py
index 802d74596d365fb6f98d3c88efe3a809c1299ac7..1a4cc699baec4cebd5d4f0fd768116ef8bc5a6b6 100644
--- a/webservice/job_monitor.py
+++ b/webservice/job_monitor.py
@@ -3,6 +3,7 @@ import argparse
import json
import locale
import logging
+import signal
import time
from datetime import datetime, timezone
from pathlib import Path
@@ -84,116 +85,229 @@ def slurm_job_status(jobid):
return "NA", "NA", "NA"
-def update_job_db(config):
- """ Update the job database and send out updates to MDC
+class JobsMonitor:
+ def __init__(self, config):
+ log.info("Starting jobs monitor")
+ self.job_db = init_job_db(config)
+ self.mdc = init_md_client(config)
+ self.kafka_prod = init_kafka_producer(config)
+ self.kafka_topic = config['kafka']['topic']
+ self.time_interval = int(config['web-service']['job-update-interval'])
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.job_db.close()
+ self.kafka_prod.close(timeout=5)
+
+ def run(self):
+ while True:
+ try:
+ self.do_updates()
+ except Exception:
+ log.error("Failure to update job DB", exc_info=True)
+ time.sleep(self.time_interval)
+
+ def do_updates(self):
+ ongoing_jobs_by_exn = self.get_updates_by_exec_id()
+
+ # For executions still running, regroup the jobs by request
+ # (by run, for correction requests):
+ reqs_still_going = {}
+ for exec_id, running_jobs_info in ongoing_jobs_by_exn.items():
+ if running_jobs_info:
+ req_id = self.job_db.execute(
+ "SELECT req_id FROM executions WHERE exec_id = ?", (exec_id,)
+ ).fetchone()[0]
+ reqs_still_going.setdefault(req_id, []).extend(running_jobs_info)
+
+ # For executions that have finished, send out notifications, and
+ # check if the whole request (several executions) has finished.
+ reqs_finished = set()
+ for exec_id, running_jobs_info in ongoing_jobs_by_exn.items():
+ if not running_jobs_info:
+ req_id = self.process_execution_finished(exec_id)
+
+ if req_id not in reqs_still_going:
+ reqs_finished.add(req_id)
+
+ # Now send updates for all requests which hadn't already finished
+ # by the last time this ran:
+
+ for req_id, running_jobs_info in reqs_still_going.items():
+ self.process_request_still_going(req_id, running_jobs_info)
+
+ for req_id in reqs_finished:
+ self.process_request_finished(req_id)
+
+ def get_updates_by_exec_id(self) -> dict:
+ """Get statuses of unfinished jobs, grouped by execution ID
+
+ E.g. {12345: ['R-5:41', 'PD-0:00', ...]}
+
+ Newly completed executions are present with an empty list.
+ """
+ c = self.job_db.cursor()
+ c.execute("SELECT job_id, exec_id FROM slurm_jobs WHERE finished = 0")
- :param config: configuration parsed from webservice YAML
- """
- log.info("Starting jobs monitor")
- conn = init_job_db(config)
- mdc = init_md_client(config)
- kafka_prod = init_kafka_producer(config)
- kafka_topic = config['kafka']['topic']
- time_interval = int(config['web-service']['job-update-interval'])
-
- while True:
statii = slurm_status()
# Check that slurm is giving proper feedback
if statii is None:
- time.sleep(time_interval)
- continue
- try:
- c = conn.cursor()
- c.execute("SELECT * FROM jobs WHERE status IN ('R', 'PD', 'CG') ")
- combined = {}
- log.debug(f"SLURM info {statii}")
-
- for r in c.fetchall():
- rid, jobid, proposal, run, status, _time, det, action = r
- log.debug(f"DB info {r}")
-
- cflg, cstatus, *_ = combined.setdefault((rid, action), (
- [], [], proposal, run, det
- ))
- if jobid in statii:
- slstatus, runtime = statii[jobid]
- query = "UPDATE jobs SET status=?, time=? WHERE jobid LIKE ?"
- c.execute(query, (slstatus, runtime, jobid))
-
- cflg.append('R')
- cstatus.append(f"{slstatus}-{runtime}")
- else:
- _, sltime, slstatus = slurm_job_status(jobid)
- query = "UPDATE jobs SET status=? WHERE jobid LIKE ?"
- c.execute(query, (slstatus, jobid))
-
- if slstatus == 'COMPLETED':
- cflg.append("A")
- else:
- cflg.append("NA")
- cstatus.append(slstatus)
- conn.commit()
-
- flg_order = {"R": 2, "A": 1, "NA": 0}
- dark_flags = {'NA': 'E', 'R': 'IP', 'A': 'F'}
- for rid, action in combined:
- if int(rid) == 0: # this job was not submitted from MyMDC
- continue
- flgs, statii, proposal, run, det = combined[rid, action]
- # sort by least done status
- flg = max(flgs, key=lambda i: flg_order[i])
- if flg != 'R':
- log.info(
- "Jobs finished - action: %s, run id: %s, status: %s",
- action, rid, flg,
- )
- if action == 'CORRECT':
- try:
- kafka_prod.send(kafka_topic, {
- 'event': 'correction_complete',
- 'proposal': proposal,
- 'run': run,
- 'detector': det,
- 'success': (flg == 'A'), # A for Available
- })
- except KafkaError:
- log.warning("Error sending Kafka notification",
- exc_info=True)
-
- if all(s.startswith('PD-') for s in statii):
- # Avoid swamping myMdC with updates for jobs still pending.
- log.debug(
- "No update for action %s, rid %s: jobs pending",
- action, rid
- )
- continue
-
- msg = "\n".join(statii)
- msg_debug = f"Update MDC {rid}, {msg}"
- log.debug(msg_debug.replace('\n', ', '))
-
- if action == 'CORRECT':
- data = {'flg_cal_data_status': flg,
- 'cal_pipeline_reply': msg}
- if flg != 'R':
- data['cal_last_end_at'] = datetime.now(tz=timezone.utc).isoformat()
- response = mdc.update_run_api(rid, data)
-
- else: # action == 'DARK' but it's dark_request
- data = {'dark_run': {'flg_status': dark_flags[flg],
- 'calcat_feedback': msg}}
- response = mdc.update_dark_run_api(rid, data)
-
- if response.status_code != 200:
- log.error("Failed to update MDC for action %s, rid %s",
- action, rid)
- log.error(Errors.MDC_RESPONSE.format(response))
- except Exception:
- log.error("Failure to update job DB", exc_info=True)
-
- time.sleep(time_interval)
-
-
+ return {}
+ log.debug(f"SLURM info {statii}")
+
+ ongoing_jobs_by_exn = {}
+ for r in c.fetchall():
+ log.debug(f"Job in DB before update: %s", tuple(r))
+ execn_ongoing_jobs = ongoing_jobs_by_exn.setdefault(r['exec_id'], [])
+
+ if str(r['job_id']) in statii:
+ # statii contains jobs which are still going (from squeue)
+ slstatus, runtime = statii[str(r['job_id'])]
+ finished = False
+ execn_ongoing_jobs.append(f"{slstatus}-{runtime}")
+
+ else:
+ # These jobs have finished (successfully or otherwise)
+ _, runtime, slstatus = slurm_job_status(r['job_id'])
+ finished = True
+
+ c.execute(
+ "UPDATE slurm_jobs SET finished=?, elapsed=?, status=? WHERE job_id = ?",
+ (finished, runtime, slstatus, r['job_id'])
+ )
+
+ self.job_db.commit()
+ return ongoing_jobs_by_exn
+
+ def process_request_still_going(self, req_id, running_jobs_info):
+ """Send myMdC updates for a request with jobs still running/pending"""
+ mymdc_id, action = self.job_db.execute(
+ "SELECT mymdc_id, action FROM requests WHERE req_id = ?",
+ (req_id,)
+ ).fetchone()
+
+ if all(s.startswith('PD-') for s in running_jobs_info):
+ # Avoid swamping myMdC with updates for jobs still pending.
+ log.debug("No update for %s request with mymdc id %s: jobs pending",
+ action, mymdc_id)
+ return
+
+ msg = "\n".join(running_jobs_info)
+ log.debug("Update MDC for %s, %s: %s",
+ action, mymdc_id, ', '.join(running_jobs_info)
+ )
+
+ if action == 'CORRECT':
+ self.mymdc_update_run(mymdc_id, msg)
+ else: # action == 'DARK'
+ self.mymdc_update_dark(mymdc_id, msg)
+
+ def process_execution_finished(self, exec_id):
+ """Send notification & record that one execution has finished"""
+ statuses = [r[0] for r in self.job_db.execute(
+ "SELECT status FROM slurm_jobs WHERE exec_id = ?", (exec_id,)
+ ).fetchall()]
+ success = set(statuses) == {'COMPLETED'}
+ r = self.job_db.execute(
+ "SELECT det_type, karabo_id, req_id, proposal, run, action "
+ "FROM executions JOIN requests USING (req_id)"
+ "WHERE exec_id = ?",
+ (exec_id,)
+ ).fetchone()
+ with self.job_db:
+ self.job_db.execute(
+ "UPDATE executions SET success = ? WHERE exec_id = ?",
+ (success, exec_id)
+ )
+ log.info("Execution finished: %s for (p%s, r%s, %s), success=%s",
+ r['action'], r['proposal'], r['run'], r['karabo_id'], success)
+
+ if r['action'] == 'CORRECT':
+ try:
+ self.kafka_prod.send(self.kafka_topic, {
+ 'event': 'correction_complete',
+ 'proposal': r['proposal'],
+ 'run': str(r['run']),
+ 'detector': r['det_type'],
+ 'detector_identifier': r['karabo_id'],
+ 'success': success,
+ })
+ except KafkaError:
+ log.warning("Error sending Kafka notification",
+ exc_info=True)
+
+ return r['req_id']
+
+ def process_request_finished(self, req_id):
+ """Send Kafka notifications and update myMDC that a request has finished."""
+ krb_id_successes = {r[0]: r[1] for r in self.job_db.execute(
+ "SELECT karabo_id, success FROM executions WHERE req_id = ?",
+ (req_id,)
+ ).fetchall()}
+ success = (set(krb_id_successes.values()) == {1})
+
+ r = self.job_db.execute(
+ "SELECT * FROM requests WHERE req_id = ?", (req_id,)
+ ).fetchone()
+ log.info(
+ "Jobs finished - action: %s, myMdC id: %s, success: %s",
+ r['action'], r['mymdc_id'], success,
+ )
+ if r['action'] == 'CORRECT':
+ try:
+ self.kafka_prod.send(self.kafka_topic, {
+ 'event': 'run_corrections_complete',
+ 'proposal': r['proposal'],
+ 'run': str(r['run']),
+ 'success': success,
+ })
+ except KafkaError:
+ log.warning("Error sending Kafka notification",
+ exc_info=True)
+
+ if success:
+ msg = "Calibration jobs succeeded"
+ else:
+ # List success & failure by karabo_id
+ krb_ids_ok = {k for (k, v) in krb_id_successes.items() if v == 1}
+ ok = ', '.join(sorted(krb_ids_ok)) if krb_ids_ok else 'none'
+ krb_ids_failed = {k for (k, v) in krb_id_successes.items() if v == 0}
+ msg = f"Succeeded: {ok}; Failed: {', '.join(sorted(krb_ids_failed))}"
+
+ log.debug("Update MDC for %s, %s: %s", r['action'], r['mymdc_id'], msg)
+
+ if r['action'] == 'CORRECT':
+ status = 'A' if success else 'NA' # Not-/Available
+ self.mymdc_update_run(r['mymdc_id'], msg, status)
+ else: # r['action'] == 'DARK'
+ status = 'F' if success else 'E' # Finished/Error
+ self.mymdc_update_dark(r['mymdc_id'], msg, status)
+
+ def mymdc_update_run(self, run_id, msg, status='R'):
+ """Update correction status in MyMdC"""
+ data = {'flg_cal_data_status': status,
+ 'cal_pipeline_reply': msg}
+ if status != 'R':
+ data['cal_last_end_at'] = datetime.now(tz=timezone.utc).isoformat()
+ response = self.mdc.update_run_api(run_id, data)
+ if response.status_code != 200:
+ log.error("Failed to update MDC run id %s", run_id)
+ log.error(Errors.MDC_RESPONSE.format(response))
+
+ def mymdc_update_dark(self, dark_run_id, msg, status='IP'):
+ """Update dark run status in MyMdC"""
+ data = {'dark_run': {'flg_status': status,
+ 'calcat_feedback': msg}}
+ response = self.mdc.update_dark_run_api(dark_run_id, data)
+
+ if response.status_code != 200:
+ log.error("Failed to update MDC dark run id %s", dark_run_id)
+ log.error(Errors.MDC_RESPONSE.format(response))
+
+def interrupted(signum, frame):
+ raise KeyboardInterrupt
def main(argv=None):
# Ensure files are opened as UTF-8 by default, regardless of environment.
@@ -218,7 +332,17 @@ def main(argv=None):
level=getattr(logging, args.log_level),
format=fmt
)
- update_job_db(config)
+ # DEBUG logs from kafka-python are very verbose, so we'll turn them off
+ logging.getLogger('kafka').setLevel(logging.INFO)
+
+ # Treat SIGTERM like SIGINT (Ctrl-C) & do a clean shutdown
+ signal.signal(signal.SIGTERM, interrupted)
+
+ with JobsMonitor(config) as jm:
+ try:
+ jm.run()
+ except KeyboardInterrupt:
+ logging.info("Shutting down on SIGINT/SIGTERM")
if __name__ == "__main__":
diff --git a/webservice/messages.py b/webservice/messages.py
index dfd7b4b487a4519abb628b496c2ba33fecb1073a..540232e41ee5f130e8c3fca722a92befbb4c2587 100644
--- a/webservice/messages.py
+++ b/webservice/messages.py
@@ -39,5 +39,6 @@ class Success:
START_CORRECTION_SIM = "SUCCESS: Simulated, not submitting jobs. Would do correction: proposal {}, run {}"
START_CHAR_SIM = "SUCCESS: Simulated, not submitting jobs. Would do dark characterization: proposal {}, run {}"
QUEUED = "SUCCESS: Queued proposal {}, run {} for offline calibration"
+ REPROD_QUEUED = "SUCCESS: Queued proposal {}, run {} for reproducing previous offline calibration"
DONE_CORRECTION = "SUCCESS: Finished correction: proposal {}. run {}"
DONE_CHAR = "SUCCESS: Finished dark characterization: proposal {}, run {}"
diff --git a/webservice/request_repeat.py b/webservice/request_repeat.py
new file mode 100644
index 0000000000000000000000000000000000000000..0851f99d5ca2b9164450318238fe65d431a56fb9
--- /dev/null
+++ b/webservice/request_repeat.py
@@ -0,0 +1,33 @@
+"""Send a request to repeat previous corrections.
+
+The repeat mechanism is meant for if corrected data has been deleted,
+but this script can also be used for testing.
+"""
+
+import argparse
+from glob import glob
+
+import zmq
+
+parser = argparse.ArgumentParser(description='Request repeat correction.')
+parser.add_argument('proposal', type=int, help='The proposal number')
+parser.add_argument('run', type=int, help='The run number')
+parser.add_argument('--mymdc-id', type=int, default=0, help="Run ID in myMdC")
+parser.add_argument('--endpoint', default='tcp://max-exfl016:5555',
+ help="The ZMQ endpoint to connect to (max-exfl017 for testing)")
+
+args = parser.parse_args()
+
+prop_dir = glob('/gpfs/exfel/exp/*/*/p{:06d}'.format(args.proposal))[0]
+instrument, cycle = prop_dir.split('/')[4:6]
+
+con = zmq.Context()
+socket = con.socket(zmq.REQ)
+con = socket.connect(args.endpoint)
+
+parm_list = ["recorrect", str(args.mymdc_id), instrument, cycle,
+ f'{args.proposal:06d}', str(args.run)]
+
+socket.send(repr(parm_list).encode())
+resp = socket.recv()
+print(resp.decode())
diff --git a/webservice/serve_overview.py b/webservice/serve_overview.py
index 2fdf7abeed59176fb1c86b080d2b51b0c1fb46c2..2aaf6fecc6f172e3952290234c3299d89b460be5 100644
--- a/webservice/serve_overview.py
+++ b/webservice/serve_overview.py
@@ -1,6 +1,7 @@
import argparse
import glob
import os
+import shlex
import sqlite3
from datetime import datetime, timezone
from http.server import BaseHTTPRequestHandler, HTTPServer
@@ -27,7 +28,6 @@ class RequestHandler(BaseHTTPRequestHandler):
def init_config(self):
self.total_jobs_cmd = config["shell-commands"]["total-jobs"]
self.tail_log_cmd = config["shell-commands"]["tail-log"]
- self.cat_log_cmd = config["shell-commands"]["cat-log"]
self.run_candidates = config["run-candidates"]
self.templates = {}
@@ -35,6 +35,8 @@ class RequestHandler(BaseHTTPRequestHandler):
with open(tfile, "r") as tf:
self.templates[template] = tf.read()
+ self.jobs_db = sqlite3.connect(config['web-service']['job-db'])
+
self.conf_was_init = True
def serve_css(self):
@@ -158,130 +160,31 @@ class RequestHandler(BaseHTTPRequestHandler):
service="Job monitor", lines=last_n_lines_monitor
)
- last_n_lines = check_output(self.cat_log_cmd,
- shell=True).decode('utf8').split("\n")[
- ::-1]
-
- def get_run_info(l, key):
- """
- Parse a line and extract information
- :param l: Line to parse
- :param key: A key work: DARK or CORRECT
- :return: Detector name, Instrument name, input folder,
- output folder, path to report, list of runs, time of request
- """
- if key in l:
- ls = l.split()
- # karabo-id and report-to are expected to be
- # in the log file
- if any(x not in ls for x in
- [key, '--karabo-id', '--report-to']):
- return None
-
- dclass = ls[ls.index(key) - 1]
- in_folder = ls[ls.index("--in-folder") + 1]
- report_to = ls[ls.index("--report-to") + 1]
- out_folder = ls[ls.index("--out-folder") + 1]
- detector = ls[ls.index("--karabo-id") + 1]
- instrument = in_folder.split('/')[4]
-
- runs = []
- for rc in self.run_candidates:
- if rc in ls:
- runs.append(ls[ls.index(rc) + 1])
-
- requested = "{} {}".format(ls[0], ls[1])
-
- return [dclass, detector, instrument, in_folder, out_folder,
- report_to, runs, requested]
-
- last_chars = {}
- last_calib = {}
host = config["server-config"]["host"]
port = config["server-config"]["port"]
- for l in last_n_lines:
- # Extract dark requests
- info = get_run_info(l, 'DARK')
- if info is not None:
- dclass, detector, instrument, in_folder, out_folder, report_to, runs, requested = info # noqa
-
- # Check if instrument is in detector name
- # This is not the case for e.g. CALLAB
- key = detector if instrument in detector else f"{instrument}-{detector}" # noqa
- if key in last_chars:
- continue
-
- pdfs = glob.glob(f"{report_to}*.pdf")
- pdfs.sort()
- pdfs = {p.split("/")[-1]: p for p in pdfs}
- fpdfs = []
- if len(pdfs):
- for pdf, p in pdfs.items():
- fpdfs.append(
- (pdf, f"http://{host}:{port}/{p}"))
- pdfs = fpdfs
- tsize = 0
- for run in runs:
- run = int(run)
- if detector not in cal_config['data-mapping']:
- continue
- # ToDo calculate tsize based on selected karabo-da
- for mp in cal_config['data-mapping'][detector]['karabo-da']:
- for f in glob.glob(
- f"{in_folder}/r{run:04d}/*{mp}*.h5"):
- tsize += os.stat(f).st_size
- constant_valid_from = ""
-
- last_chars[key] = {"in_path": in_folder,
- "out_path": out_folder,
- "runs": runs,
- "pdfs": pdfs,
- "size": "{:0.1f} GB".format(tsize / 1e9),
- "requested": requested,
- "device_type": detector,
- "last_valid_from": constant_valid_from}
-
- # Extract correction requests
- info = get_run_info(l, 'CORRECT')
- if info is not None:
- _, _, _, in_folder, _, report_to, runs, requested = info
- instrument = in_folder.split('/')[4]
- if instrument not in last_calib:
- last_calib[instrument] = []
- if len(last_calib[instrument]) > config["server-config"]["n-calib"]: # noqa
- continue
- proposal = in_folder.split('/')[6]
- pdfs = glob.glob(f"{report_to}*.pdf")
- pdfs.sort(key=os.path.getmtime, reverse=True)
- pdfs = {p.split("/")[-1]: p for p in pdfs}
-
- if [proposal, runs[0]] not in [[x[1], x[2]] for x in
- last_calib[instrument]]:
- last_calib[instrument].append([requested[:-4],
- proposal, runs[0], pdfs])
tmpl = self.templates["last-characterizations"]
- last_characterizations_r = Template(tmpl).render(char_runs=last_chars,
- host = host,
- port = port)
+ last_characterizations_r = Template(tmpl).render(
+ char_runs=self.get_last_chars(), host=host, port=port
+ )
tmpl = self.templates["last-correction"]
- last_correction_r = Template(tmpl).render(info=last_calib, host=host,
- port=port)
+ last_correction_r = Template(tmpl).render(
+ info=self.get_last_corrections(), host=host, port=port
+ )
- conn = sqlite3.connect(config['web-service']['job-db']).cursor()
- conn.execute("SELECT * FROM jobs WHERE status IN ('R', 'PD', 'CG')")
+ c = self.jobs_db.execute(
+ "SELECT status, elapsed, det_type, proposal, run, action FROM "
+ "slurm_jobs INNER JOIN executions USING (exec_id) "
+ "INNER JOIN requests USING (req_id) "
+ "WHERE finished = 0"
+ )
running_jobs = {}
- for r in conn.fetchall():
- rid, jobid, proposal, run, status, time, det, act = r
- run = int(run)
- key = '{}/r{:04d}/{}/{}'.format(proposal, run, det, act)
- flg = "R"
- if status in ["QUEUE", "PD"]:
- flg = "Q"
- rjobs = running_jobs.get(key, [])
- rjobs.append((flg, '{}-{}'.format(status, time)))
- running_jobs[key] = rjobs
+ for status, elapsed, det, proposal, run, act in c:
+ key = f'{proposal}/r{int(run):04d}/{det}/{act}'
+ flg = "Q" if status in {"QUEUE", "PD"} else "R"
+ rjobs = running_jobs.setdefault(key, [])
+ rjobs.append((flg, f'{status}-{elapsed}'))
tmpl = self.templates["running-jobs"]
running_jobs_r = Template(tmpl).render(running_jobs=running_jobs)
@@ -297,6 +200,107 @@ class RequestHandler(BaseHTTPRequestHandler):
self.wfile.write(bytes(message, "utf8"))
return
+ def parse_calibrate_command(self, cmd):
+ args = shlex.split(cmd)
+
+ in_folder = args[args.index("--in-folder") + 1]
+ report_to = args[args.index("--report-to") + 1]
+ out_folder = args[args.index("--out-folder") + 1]
+ detector = args[args.index("--karabo-id") + 1]
+ instrument = in_folder.split('/')[4]
+
+ runs = []
+ for rc in self.run_candidates:
+ if rc in args:
+ runs.append(args[args.index(rc) + 1])
+
+ return detector, instrument, in_folder, out_folder, report_to, runs
+
+ def get_last_chars(self):
+ cur = self.jobs_db.execute("""
+ SELECT command, det_type, karabo_id, timestamp
+ FROM executions INNER JOIN requests USING (req_id)
+ WHERE action = 'DARK'
+ ORDER BY timestamp DESC
+ LIMIT 100
+ """)
+
+ last_chars = {}
+ for command, det_type, karabo_id, timestamp in cur:
+ try:
+ detector, instrument, in_folder, out_folder, report_to, runs = \
+ self.parse_calibrate_command(command)
+ except Exception as e:
+ print("Failed parsing xfel-calibrate command", e, flush=True)
+ continue
+
+ key = detector if instrument in detector else f"{instrument}-{detector}" # noqa
+
+ # Check if instrument is in detector name
+ # This is not the case for e.g. CALLAB
+ key = detector if instrument in detector else f"{instrument}-{detector}" # noqa
+ if key in last_chars:
+ continue
+
+ pdfs = [
+ (os.path.basename(p), p)
+ for p in sorted(glob.glob(f"{report_to}*.pdf"))
+ ]
+ tsize = 0
+ for run in runs:
+ run = int(run)
+ if detector not in cal_config['data-mapping']:
+ continue
+ # ToDo calculate tsize based on selected karabo-da
+ for mp in cal_config['data-mapping'][detector]['karabo-da']:
+ for f in glob.glob(
+ f"{in_folder}/r{run:04d}/*{mp}*.h5"):
+ tsize += os.stat(f).st_size
+
+ last_chars[key] = {"in_path": in_folder,
+ "out_path": out_folder,
+ "runs": runs,
+ "pdfs": pdfs,
+ "size": "{:0.1f} GB".format(tsize / 1e9),
+ "requested": timestamp,
+ "device_type": detector,
+ }
+
+ return last_chars
+
+ def get_last_corrections(self):
+ cur = self.jobs_db.execute("""
+ SELECT command, det_type, karabo_id, proposal, timestamp
+ FROM executions INNER JOIN requests USING (req_id)
+ WHERE action = 'CORRECT'
+ ORDER BY timestamp DESC
+ LIMIT 100
+ """)
+
+ last_calib = {}
+ for command, det_type, karabo_id, proposal, timestamp in cur:
+ try:
+ detector, instrument, in_folder, out_folder, report_to, runs = \
+ self.parse_calibrate_command(command)
+ except Exception as e:
+ print("Failed parsing xfel-calibrate command", e, flush=True)
+ continue
+
+ inst_records = last_calib.setdefault(instrument, [])
+ if len(inst_records) >= config["server-config"]["n-calib"]: # noqa
+ continue
+
+ pdfs = glob.glob(f"{report_to}*.pdf")
+ pdfs.sort(key=os.path.getmtime, reverse=True)
+ pdfs = {p.split("/")[-1]: p for p in pdfs}
+
+ if not any(r[1:3] == (proposal, runs[0]) for r in inst_records):
+ inst_records.append((
+ timestamp[:-4], proposal, runs[0], pdfs
+ ))
+
+ return last_calib
+
def run(config_file: Optional[str] = None):
if config_file is not None:
@@ -305,11 +309,11 @@ def run(config_file: Optional[str] = None):
with open(config["web-service"]["cal-config"], "r") as cf:
global cal_config
cal_config = yaml.load(cf.read(), Loader=yaml.FullLoader)
- print('starting server...')
+ print('starting server...', flush=True)
sconfig = config["server-config"]
server_address = (sconfig["host"], sconfig["port"])
httpd = HTTPServer(server_address, RequestHandler)
- print('running server...')
+ print('running server...', flush=True)
httpd.serve_forever()
diff --git a/webservice/sqlite_view.py b/webservice/sqlite_view.py
index 29670a19f3b02b710da98e713bf9f176e56809d4..b797afe9c36371161894ec047c729db5d573858b 100644
--- a/webservice/sqlite_view.py
+++ b/webservice/sqlite_view.py
@@ -2,12 +2,11 @@ import argparse
import sqlite3
parser = argparse.ArgumentParser(
- description='Update run status at MDC for a given run id.')
+ description='Check jobs for a given proposal & run number')
parser.add_argument('--sqlite-fpath', type=str, help='Path to sqlite file path',
- default='/home/xcal/calibration_webservice/webservice/webservice_jobs.sqlite') # noqa
-parser.add_argument('--run', type=str, help='The run number required '
- ' for checking its job status.')
-parser.add_argument('--proposal', type=str, help='Proposal numer')
+ default='webservice_jobs.sqlite')
+parser.add_argument('--proposal', type=str, required=True, help='Proposal number')
+parser.add_argument('--run', type=int, required=True, help='Run number')
args = vars(parser.parse_args())
@@ -16,11 +15,14 @@ proposal = args['proposal'].zfill(6)
run = args['run']
conn = sqlite3.connect(sqlite_fpath)
-c = conn.cursor()
-c.execute("SELECT * FROM jobs")
+c = conn.execute(
+ "SELECT status, elapsed, karabo_id, det_type, action FROM "
+ "slurm_jobs INNER JOIN executions USING (exec_id) "
+ "INNER JOIN requests USING (req_id) "
+ "WHERE proposal = ? AND run = ?",
+ (proposal, run)
+)
for r in c.fetchall():
- rid, jobid, db_proposal, db_run, status, time, _, _ = r
- if db_proposal == proposal and db_run == run:
- print(r)
+ print(r)
diff --git a/webservice/update_config.py b/webservice/update_config.py
index 5e5960e370cf1e1053c4c0c571c0650fd4ca9f70..f6a6f9d1f9bb7c7474724d955fd359f3f4fa2d5f 100755
--- a/webservice/update_config.py
+++ b/webservice/update_config.py
@@ -9,37 +9,40 @@ import zmq
AGIPD_CONFIGURATIONS = {
"correct":
{
- "force-hg-if-below": {'typ': int},
- "rel-gain": {'typ': bool},
- "xray-gain": {'typ': bool},
- "blc-noise": {'typ': bool},
- "blc-set-min": {'typ': bool},
- "dont-zero-nans": {'typ': bool},
- "dont-zero-orange": {'typ': bool},
- "max-pulses": {'typ': list,
+ "force-hg-if-below": {'type': int},
+ "rel-gain": {'type': bool},
+ "xray-gain": {'type': bool},
+ "blc-noise": {'type': bool},
+ "blc-set-min": {'type': bool},
+ "dont-zero-nans": {'type': bool},
+ "dont-zero-orange": {'type': bool},
+ "max-pulses": {'type': list,
'msg': "Range list of maximum pulse indices "
"(--max-pulses start end step). "
"3 max input elements. "},
+ 'use-litframe-finder': {'type': str},
+ 'litframe-device-id': {'type': str},
+ 'energy-threshold': {'type': int}
},
"dark":
{
- "thresholds-offset-hard-hg": {'typ': list},
- "thresholds-offset-hard-mg": {'typ': list},
- "thresholds-offset-hard-lg": {'typ': list},
- "thresholds-offset-hard-hg-fixed": {'typ': list},
- "thresholds-offset-hard-mg-fixed": {'typ': list},
- "thresholds-offset-hard-lg-fixed": {'typ': list},
- "thresholds-offset-sigma": {'typ': list},
- "thresholds-noise-hard-hg": {'typ': list},
- "thresholds-noise-hard-mg": {'typ': list},
- "thresholds-noise-hard-lg": {'typ': list},
- "thresholds-noise-sigma": {'typ': list},
+ "thresholds-offset-hard-hg": {'type': list},
+ "thresholds-offset-hard-mg": {'type': list},
+ "thresholds-offset-hard-lg": {'type': list},
+ "thresholds-offset-hard-hg-fixed": {'type': list},
+ "thresholds-offset-hard-mg-fixed": {'type': list},
+ "thresholds-offset-hard-lg-fixed": {'type': list},
+ "thresholds-offset-sigma": {'type': list},
+ "thresholds-noise-hard-hg": {'type': list},
+ "thresholds-noise-hard-mg": {'type': list},
+ "thresholds-noise-hard-lg": {'type': list},
+ "thresholds-noise-sigma": {'type': list},
}
}
DATA_MAPPING = {
"karabo-da": {
- 'typ': list,
+ 'type': list,
'choices': [f"AGIPD{i:02d}" for i in range(16)],
'msg': "Choices: [AGIPD00 ... AGIPD15]. "
}
@@ -108,19 +111,19 @@ def _add_available_configs_to_arg_parser(karabo_id: str, action: str):
# Loop over action configurations in available_detectors dictionary.
for key, val in AVAILABLE_DETECTORS[karabo_id][0][action].items():
available_conf[0][key] = val
- if val['typ'] == bool:
- available_conf[0][f'no-{key}'] = {'typ': bool}
+ if val['type'] == bool:
+ available_conf[0][f'no-{key}'] = {'type': bool}
for conf in available_conf:
for option, info in conf.items():
- typ = info['typ']
+ type_ = info['type']
choices = info.get('choices')
- if info['typ'] == list:
+ if info['type'] == list:
arguments = {
"action": 'append',
"nargs": '+',
}
- typ = str
+ type_ = str
# Avoid having a big line of choices in the help message.
if choices:
arguments.update({
@@ -135,10 +138,10 @@ def _add_available_configs_to_arg_parser(karabo_id: str, action: str):
if 'msg' in info.keys():
help_msg += info['msg']
- help_msg += f"Type: {info['typ'].__name__} ".upper()
+ help_msg += f"Type: {info['type'].__name__} ".upper()
parser.add_argument(
f"--{option}",
- type=typ,
+ type=type_,
help=help_msg,
**arguments,
)
diff --git a/webservice/webservice.py b/webservice/webservice.py
index 27407f6fdd2eccccdb63af1cee1d1154f380beaa..394db06c3a722279fd30a5f153f0bfc15ca39764 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -1,6 +1,7 @@
import argparse
import ast
import asyncio
+import contextlib
import copy
import glob
import inspect
@@ -8,11 +9,13 @@ import json
import locale
import logging
import os
+import shlex
import sqlite3
import sys
import urllib.parse
from asyncio import get_event_loop, shield
from datetime import datetime, timezone
+from getpass import getuser
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union
@@ -45,10 +48,37 @@ def init_job_db(config):
"""
logging.info("Initializing database")
conn = sqlite3.connect(config['web-service']['job-db'])
- conn.execute(
- "CREATE TABLE IF NOT EXISTS "
- "jobs(rid, jobid, proposal, run, status, time, det, act)"
- )
+ conn.execute("PRAGMA foreign_keys = ON")
+
+ conn.executescript("""
+ CREATE TABLE IF NOT EXISTS requests(
+ req_id INTEGER PRIMARY KEY,
+ mymdc_id,
+ proposal TEXT,
+ run INTEGER,
+ action,
+ timestamp
+ );
+ CREATE TABLE IF NOT EXISTS executions(
+ exec_id INTEGER PRIMARY KEY,
+ req_id REFERENCES requests(req_id),
+ command TEXT,
+ det_type,
+ karabo_id,
+ success
+ );
+ CREATE INDEX IF NOT EXISTS exec_by_req ON executions(req_id);
+ CREATE TABLE IF NOT EXISTS slurm_jobs(
+ job_id INTEGER PRIMARY KEY,
+ exec_id REFERENCES executions(exec_id),
+ status,
+ finished,
+ elapsed
+ );
+ CREATE INDEX IF NOT EXISTS job_by_exec ON slurm_jobs(exec_id);
+ CREATE INDEX IF NOT EXISTS job_by_finished ON slurm_jobs(finished);
+ """)
+ conn.row_factory = sqlite3.Row
return conn
@@ -227,37 +257,20 @@ async def run_proc_async(cmd: List[str]) -> Tuple[Optional[int], bytes, bytes]:
def query_rid(conn, rid) -> bytes:
- c = conn.cursor()
- c.execute("SELECT * FROM jobs WHERE rid LIKE ?", (rid,))
- combined = {}
- for r in c.fetchall():
- rid, jobid, proposal, run, status, time_, _ = r
- logging.debug(
- "Job {}, proposal {}, run {} has status {}".format(jobid,
- proposal,
- run,
- status))
- cflg, cstatus = combined.get(rid, ([], []))
- if status in ['R', 'PD']:
- flg = 'R'
- elif status == 'NA':
- flg = 'NA'
- else:
- flg = 'A'
-
- cflg.append(flg)
- cstatus.append(status)
- combined[rid] = cflg, cstatus
+ c = conn.execute(
+ "SELECT job_id, status FROM slurm_jobs "
+ "INNER JOIN executions USING (exec_id) "
+ "INNER JOIN requests USING (req_id) "
+ "WHERE mymdc_id = ?", (rid,)
+ )
+ statuses = []
+ for job_id, status in c.fetchall():
+ logging.debug("Job %s has status %s", job_id, status)
+ statuses.append(status)
- flg_order = {"R": 2, "A": 1, "NA": 0}
- msg = ""
- for rid, value in combined.items():
- flgs, statii = value
- flg = max(flgs, key=lambda i: flg_order[i])
- msg += "\n".join(statii)
- if msg == "":
- msg = 'NA'
- return msg.encode()
+ if statuses:
+ return "\n".join(statuses).encode()
+ return b'NA'
def parse_config(cmd: List[str], config: Dict[str, Any]) -> List[str]:
@@ -289,7 +302,7 @@ def parse_config(cmd: List[str], config: Dict[str, Any]) -> List[str]:
-async def run_action(job_db, cmd, mode, proposal, run, rid) -> str:
+async def run_action(job_db, cmd, mode, proposal, run, exec_id) -> str:
"""Run action command (CORRECT or DARK).
:param job_db: jobs database
@@ -298,7 +311,7 @@ async def run_action(job_db, cmd, mode, proposal, run, rid) -> str:
but the command will be logged
:param proposal: proposal the command was issued for
:param run: run the command was issued for
- :param rid: run id in the MDC
+ :param exec_id: Execution ID in the local jobs database
Returns a formatted Success or Error message indicating outcome of the
execution.
@@ -310,7 +323,7 @@ async def run_action(job_db, cmd, mode, proposal, run, rid) -> str:
logging.error(Errors.JOB_LAUNCH_FAILED.format(cmd, retcode))
logging.error(f"Failed process stdout:\n%s\nFailed process stderr:\n%s",
stdout.decode(errors='replace'), stderr.decode(errors='replace'))
- return Errors.JOB_LAUNCH_FAILED.format(cmd, retcode)
+ return Errors.JOB_LAUNCH_FAILED.format("internal error", retcode)
if "DARK" in cmd:
message = Success.START_CHAR.format(proposal, run)
@@ -327,17 +340,11 @@ async def run_action(job_db, cmd, mode, proposal, run, rid) -> str:
jobs = []
for jobid in jobids.split(','):
- jobs.append((rid,
- jobid.strip(),
- proposal,
- run,
- datetime.now().isoformat(),
- cmd[3],
- cmd[4])
- )
+ jobs.append((int(jobid.strip()), exec_id))
c.executemany(
- "INSERT INTO jobs VALUES (?, ?, ?, ?, 'PD', ?, ?, ?)",
- jobs)
+ "INSERT INTO slurm_jobs VALUES (?, ?, 'PD', 0, 0)",
+ jobs
+ )
job_db.commit()
else: # mode == "sim"
@@ -499,14 +506,20 @@ def check_files(in_folder: str,
files_exists = False
return files_exists
-async def get_slurm_partition(mdc: MetadataClient,
- action: str,
- proposal_number: Union[int, str]) -> str:
+
+async def get_slurm_partition(
+ mdc: MetadataClient,
+ action: str,
+ proposal_number: Union[int, str],
+ run_id: Optional[int] = None,
+ cycle: Optional[int] = None
+) -> str:
"""Check MyMDC for the proposal status and select the appropriate slurm
partition.
The partition is either upex-high (for darks) or upex-middle (for
- corrections) if the proposal is 'R'eady or 'A'ctive.
+ corrections) if the proposal is 'R'eady or 'A'ctive, and within the
+ beamtimes.
In other cases, the jobs default to the exfel partition.
:param mdc: an authenticated MyMDC client
@@ -518,24 +531,88 @@ async def get_slurm_partition(mdc: MetadataClient,
"""
# See
# https://git.xfel.eu/ITDM/metadata_catalog/-/blob/develop/app/models/proposal.rb
- # for possible proposals states.
+ # for possible proposals states. Relevant ones for us are:
+ # | Flag | Name | Description |
+ # |------|--------------|----------------------------------------|
+ # | R | Ready | Users created, proposal structure |
+ # | | | initialised on GPFS |
+ # |------|--------------|----------------------------------------|
+ # | A | Active | Data currently being acquired |
+ # |------|--------------|----------------------------------------|
+ # | I | Inactive | Proposal still scheduled, data not |
+ # | | | currently being acquired |
+ # |------|--------------|----------------------------------------|
+ # | F | Finished | Beamtime is over, no more new runs |
+ # |------|--------------|----------------------------------------|
+ # | FR | Finished and | Local instrument contact reviewed |
+ # | | Reviewed | details after initial finish flag set |
+ # |------|--------------|----------------------------------------|
+ # | CLO | Closed | Data fully migrated to dCACHE |
+ # |------|--------------|----------------------------------------|
loop = get_event_loop()
- response = await shield(loop.run_in_executor(None,
- mdc.get_proposal_by_number_api,
- proposal_number))
+
+ response = await shield(
+ loop.run_in_executor(
+ None,
+ mdc.get_proposal_by_number_api,
+ proposal_number
+ )
+ )
+
if response.status_code != 200:
- logging.error(f'Failed to check MDC for proposal "{proposal_number}" '
- 'status. ASSUMING CLOSED')
+ logging.error(
+ f'Failed to check MDC for proposal "{proposal_number}" '
+ 'status. ASSUMING CLOSED'
+ )
logging.error(Errors.MDC_RESPONSE.format(response))
partition = 'exfel'
- status = response.json().get('flg_beamtime_status', 'whoopsie')
+ active_now = False
- if status in ['R', 'A']:
+ status_beamtime = response.json().get('flg_beamtime_status', 'whoopsie')
+
+ if status_beamtime in ['R', 'A']:
partition = 'upex-high' if action == 'dark' else 'upex-middle'
- logging.debug(f"Using {partition} for {proposal_number} because {status}")
+ # A proposal can have several beamtimes during which data can be taken
+ # that are independent from the start and end dates.
+ beamtimes = response.json().get('beamtimes', [])
+ now = datetime.now().timestamp()
+ for beamtime in beamtimes:
+ begin = datetime.fromisoformat(beamtime['begin_at']).timestamp()
+ end = datetime.fromisoformat(beamtime['end_at']).timestamp()
+ if begin <= now <= end:
+ active_now = True
+ break
+ partition = partition if active_now else 'exfel'
+
+ # NOTE: non-zero at cycle index 4 (`str(cycle)[4]`) indicates commissioning
+ if run_id and cycle and str(cycle)[4] != '0':
+ response_run = await shield(
+ loop.run_in_executor(
+ None,
+ mdc.get_run_by_id_api,
+ run_id
+ )
+ )
+
+ if (begin_at := response_run.json().get("begin_at", None)):
+ with contextlib.suppress(ValueError):
+ run_begin = datetime.strptime(begin_at, "%Y-%m-%dT%H:%M:%S.%f%z")
+ run_age = (datetime.now().astimezone() - run_begin)
+ max_age_days = config[action]["commissioning-max-age-days"]
+ if run_age.days > max_age_days:
+ partition = 'exfel'
+ logging.debug(
+ f"{run_age.days} > {max_age_days}, set partition "
+ f"to {partition}"
+ )
+
+ logging.debug(
+ f"Using {partition} for {proposal_number} because {status_beamtime} "
+ f"and active now: {active_now}"
+ )
return partition
@@ -781,6 +858,7 @@ class ActionsServer:
accepted_actions = {
'correct',
+ 'recorrect',
'dark_request',
'query-rid',
'upload-yaml',
@@ -821,6 +899,13 @@ class ActionsServer:
proposal = self._normalise_proposal_num(proposal)
pconf_full = self.load_proposal_config(cycle, proposal)
+ with self.job_db:
+ cur = self.job_db.execute(
+ "INSERT INTO requests VALUES (NULL, ?, ?, ?, 'CORRECT', ?)",
+ (rid, proposal, int(runnr), request_time)
+ )
+ req_id = cur.lastrowid
+
_orca_passthrough(
proposal_number=proposal,
runs=[runnr],
@@ -899,23 +984,124 @@ class ActionsServer:
logging.warning(f'Skipping disabled detector {karabo_id}')
del detectors[karabo_id]
- if len(detectors) == 0:
+ if not detectors:
msg = Errors.NOTHING_TO_DO.format(rpath)
logging.warning(msg)
await update_mdc_status(self.mdc, 'correct', rid, msg)
return
ret, _ = await self.launch_jobs(
- [runnr], rid, detectors, 'correct', instrument, cycle, proposal,
- request_time,
+ [runnr], req_id, detectors, 'correct', instrument, cycle,
+ proposal, request_time, rid
)
await update_mdc_status(self.mdc, 'correct', rid, ret)
loop = get_event_loop()
await loop.run_in_executor(
+ None,
+ self.mdc.update_run_api,
+ rid,
+ {'cal_last_begin_at': datetime.now(tz=timezone.utc).isoformat()}
+ )
+
+ # END of part to run after sending reply
+
+ asyncio.ensure_future(_continue())
+
+ return queued_msg.encode()
+
+ async def handle_recorrect(self, rid, instrument, cycle, proposal, runnr):
+ request_time = datetime.now()
+
+ try:
+ with self.job_db:
+ cur = self.job_db.execute(
+ "INSERT INTO requests VALUES (NULL, ?, ?, ?, 'CORRECT', ?)",
+ (rid, proposal, int(runnr), request_time.strftime('%Y-%m-%dT%H:%M:%S'))
+ )
+ req_id = cur.lastrowid
+
+ reports_dir = Path(self.config['correct']['reports-folder'].format(
+ instrument=instrument, cycle=cycle, proposal=proposal, runs=f"r{runnr}"
+ ))
+
+ mddirs_by_krb_id = {}
+ for calmeta_pth in sorted(reports_dir.glob('*/calibration_metadata.yml')):
+ with calmeta_pth.open('r', encoding='utf-8') as f:
+ calmeta = yaml.safe_load(f)
+
+ try:
+ prior_request_time = calmeta["runtime-summary"]\
+ ["pipeline-steps"]["request-time"]
+ karabo_id = calmeta["calibration-configurations"]["karabo-id"]
+ except KeyError:
+ logging.warning("Did not find expected metadata in %s",
+ calmeta_pth, exc_info=True)
+ else:
+ mddirs_by_krb_id.setdefault(karabo_id, []).append(
+ (calmeta_pth.parent, prior_request_time)
+ )
+
+ logging.info("Found %d corrections to re-run for p%s r%s: %s",
+ len(mddirs_by_krb_id), proposal, runnr, list(mddirs_by_krb_id))
+
+ except Exception as e:
+ msg = Errors.JOB_LAUNCH_FAILED.format('correct', e)
+ logging.error(msg, exc_info=e)
+ asyncio.ensure_future(
+ update_mdc_status(self.mdc, 'correct', rid, msg)
+ )
+ return msg.encode()
+
+ queued_msg = Success.REPROD_QUEUED.format(proposal, [runnr])
+ logging.debug(queued_msg)
+
+ async def _continue():
+ """Runs in the background after we reply to the ZMQ request"""
+ if len(mddirs_by_krb_id) == 0:
+ in_folder = self.config['correct']['in-folder'].format(
+ instrument=instrument, cycle=cycle, proposal=proposal)
+ rpath = os.path.join(in_folder, f"r{int(runnr):04d}/")
+ msg = Errors.NOTHING_TO_DO.format(rpath)
+ logging.warning(msg)
+ await update_mdc_status(self.mdc, 'correct', rid, msg)
+ return
+
+ await update_mdc_status(self.mdc, 'correct', rid, queued_msg)
+
+ ret = []
+ for karabo_id, mddirs in mddirs_by_krb_id.items():
+ # Select the latest metadata directory - with the highest request
+ # time - to re-run for each detector (karabo_id)
+ mddir, _ = max(mddirs, key=lambda p: p[1])
+
+ logging.info("Repeating correction for %s from %s", karabo_id, mddir)
+
+ cmd = [
+ 'python', '-m', 'xfel_calibrate.repeat', str(mddir),
+ '--env-cache',
+ f'/gpfs/exfel/data/scratch/{getuser()}/calib-repeat-envs',
+ '--report-to',
+ f'{reports_dir}/{karabo_id}_RECORRECT_{request_time:%y%m%d_%H%M%S}'
+ ]
+
+ with self.job_db:
+ cur = self.job_db.execute(
+ "INSERT INTO executions VALUES (NULL, ?, ?, NULL, ?, NULL)",
+ (req_id, shlex.join(cmd), karabo_id)
+ )
+ exec_id = cur.lastrowid
+
+ ret.append(await run_action(
+ self.job_db, cmd, self.mode,
+ proposal, runnr, exec_id
+ ))
+
+ await update_mdc_status(self.mdc, 'correct', rid, ", ".join(ret))
+ await get_event_loop().run_in_executor(
None, self.mdc.update_run_api,
rid, {'cal_last_begin_at': datetime.now(tz=timezone.utc).isoformat()}
)
- # END of part to run after sending reply
+ # END of part to run after sending reply
asyncio.ensure_future(_continue())
@@ -958,6 +1144,13 @@ class ActionsServer:
karabo_id=karabo_id,
)
+ with self.job_db:
+ cur = self.job_db.execute(
+ "INSERT INTO requests VALUES (NULL, ?, ?, ?, 'DARK', ?)",
+ (rid, proposal, int(wait_runs[-1]), request_time)
+ )
+ req_id = cur.lastrowid
+
pconf_full = self.load_proposal_config(cycle, proposal)
data_conf = pconf_full['data-mapping']
@@ -1001,7 +1194,7 @@ class ActionsServer:
# Notebooks require one or three runs, depending on the
# detector type and operation mode.
triple = any(det in karabo_id for det in
- ["LPD", "AGIPD", "JUNGFRAU", "JF", "JNGFR"])
+ ["LPD", "AGIPD", "JUNGFRAU", "JF", "JNGFR", "G2"])
# This fails silently if the hardcoded strings above are
# ever changed (triple = False) but the underlying notebook
@@ -1039,7 +1232,7 @@ class ActionsServer:
detectors = {karabo_id: thisconf}
ret, report_path = await self.launch_jobs(
- runs, rid, detectors, 'dark', instrument, cycle, proposal,
+ runs, req_id, detectors, 'dark', instrument, cycle, proposal,
request_time
)
await update_mdc_status(self.mdc, 'dark_request', rid, ret)
@@ -1127,13 +1320,16 @@ class ActionsServer:
return yaml.load(f.read(), Loader=yaml.FullLoader)
async def launch_jobs(
- self, run_nrs, rid, detectors, action, instrument, cycle, proposal,
- request_time
+ self, run_nrs, req_id, detectors, action, instrument, cycle,
+ proposal, request_time, run_id: Optional[int] = None
) -> Tuple[str, List[str]]:
report = []
ret = []
- partition = await get_slurm_partition(self.mdc, action, proposal)
+ partition = await get_slurm_partition(
+ self.mdc, action, proposal, run_id, cycle
+ )
+
nice = await get_slurm_nice(
partition, instrument, cycle,
commissioning_penalty=self.config[action]['commissioning-penalty'],
@@ -1154,12 +1350,19 @@ class ActionsServer:
det_instance=karabo_id,
request_time=request_time
).split()
-
cmd = parse_config(cmd, dconfig)
+
+ with self.job_db:
+ cur = self.job_db.execute(
+ "INSERT INTO executions VALUES (NULL, ?, ?, ?, ?, NULL)",
+ (req_id, shlex.join(cmd), detector, karabo_id)
+ )
+ exec_id = cur.lastrowid
+
# TODO: Add detector info in returned run action status.
ret.append(await run_action(
self.job_db, cmd, self.mode,
- proposal, run_nrs[-1], rid
+ proposal, run_nrs[-1], exec_id
))
if '--report-to' in cmd[:-1]:
report_idx = cmd.index('--report-to') + 1