diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
index a02c88ca8b9c9746ebc81c2e5c8f059abe18b3a9..1c8ba674c4f8e75fff4887708a49579bf47beed1 100644
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@@ -11,3 +11,7 @@ e7dfadaf4e189ef0e0f67798e8984695111257e3
# fix/requirements-into-setuppy - Reformat setup.py without content changes
31b402966a4a741d08cb1e89db96f252fbb41c54
+# fix/pre-commit-isort - Apply isort to notebooks
+604d51278aa137d151c4b693582925d1eedb7c4e
+# fix/pre-commit-isort - Apply isort to .py files
+881392a435e73478d07e37fcfecfaa804cabd0e6
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index da0c4977b26c4052c8663682a6efc5a727435354..10959e513ce6bd8b9799fcc8f21eff646624958d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,24 +3,25 @@ repos:
hooks:
- id: identity
- repo: https://github.com/nbQA-dev/nbQA
- rev: 0.3.6
+ rev: 0.5.9
hooks:
- id: nbqa-isort
- additional_dependencies: [isort==5.6.4]
+ additional_dependencies: [isort==5.7.0]
args: [--nbqa-mutate]
- id: nbqa-flake8
- additional_dependencies: [flake8==3.8.4]
+ additional_dependencies: [flake8==3.9.0]
args: [--nbqa-mutate]
+ - id: nbqa-check-ast
- repo: https://github.com/kynan/nbstripout
rev: 0.3.9
hooks:
- id: nbstripout
- repo: https://github.com/pycqa/isort
- rev: 5.6.4
+ rev: 5.7.0
hooks:
- id: isort
- repo: https://gitlab.com/pycqa/flake8
- rev: 3.8.4
+ rev: 3.9.0
hooks:
- id: flake8
# If `CI_MERGE_REQUEST_TARGET_BRANCH_SHA` env var is set then this will
@@ -33,7 +34,7 @@ repos:
hooks:
- id: rstcheck
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v2.3.0
+ rev: v3.4.0
hooks:
- id: check-added-large-files
- id: check-ast
diff --git a/README.rst b/README.rst
index 167a2ece7c628d8b783f7bd04065851ea7b26bf6..f87324e951ff9b68a309d7755f962db9eafc460d 100644
--- a/README.rst
+++ b/README.rst
@@ -81,11 +81,13 @@ This can be useful for Jupyter notebook tools as https://max-jhub.desy.de/hub/lo
Contributing
************
+
Guidelines
==========
Development guidelines can be found on the GitLab Wiki page here: https://git.xfel.eu/gitlab/detectors/pycalibration/wikis/GitLab-Guidelines
+
Basics
======
@@ -101,6 +103,7 @@ GitLab.
To do this check the settings page here: https://git.xfel.eu/gitlab/profile/keys
+
Pre-Commit Hooks
================
@@ -239,6 +242,7 @@ to provide accurate relative gain correction constants.
You'll get a series of plots in the output directory as well.
+
Appendix
********
@@ -246,6 +250,7 @@ Important information that doesn't really fit in as part of the readme.
TODO: Place this into the docs? Also, improve docs (out of scope for PR !437)
+
GitLab Access for ``xcaltst`` and ``xcal``
==========================================
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index e797a3bdea662b5ef7fa47c01dbf9f824f52073d..073c106be034cd064c4cf8d2216492ddacdcb7a2 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -5,16 +5,16 @@ from typing import Any, Dict, Optional, Tuple
import h5py
import numpy as np
import sharedmem
-from iCalibrationDB import Conditions, Constants
-
-from cal_tools.agipdutils import (assemble_constant_dict,
- baseline_correct_via_noise,
- baseline_correct_via_stripe,
- correct_baseline_via_hist,
- correct_baseline_via_hist_asic,
- make_noisy_adc_mask, match_asic_borders,
- melt_snowy_pixels)
-from cal_tools.cython import agipdalgs as calgs
+from cal_tools.agipdutils import (
+ assemble_constant_dict,
+ baseline_correct_via_noise,
+ baseline_correct_via_stripe,
+ correct_baseline_via_hist,
+ correct_baseline_via_hist_asic,
+ make_noisy_adc_mask,
+ match_asic_borders,
+ melt_snowy_pixels,
+)
from cal_tools.enums import BadPixels, SnowResolution
from cal_tools.tools import get_constant_from_db_and_time
diff --git a/cal_tools/cal_tools/agipdutils.py b/cal_tools/cal_tools/agipdutils.py
index ae06e49bf9d661a6cc90265b445c93860bfee030..9d58ccf83ba9cc692a043c64c562d78f1c1afb99 100644
--- a/cal_tools/cal_tools/agipdutils.py
+++ b/cal_tools/cal_tools/agipdutils.py
@@ -2,12 +2,11 @@ import copy
from typing import Tuple
import numpy as np
+from cal_tools.enums import BadPixels, SnowResolution
from scipy.signal import cwt, ricker
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import StandardScaler
-from cal_tools.enums import BadPixels, SnowResolution
-
def assemble_constant_dict(corr_bools, pc_bools, memory_cells, bias_voltage,
gain_setting, acquisition_rate,
diff --git a/cal_tools/cal_tools/agipdutils_ff.py b/cal_tools/cal_tools/agipdutils_ff.py
index 6774e5a0e85a619796e8d1c510e3a5a8b52852bf..4cae853275d5683c9760ca5565472d019ea56763 100644
--- a/cal_tools/cal_tools/agipdutils_ff.py
+++ b/cal_tools/cal_tools/agipdutils_ff.py
@@ -1,9 +1,8 @@
from typing import Any, Dict, List, Optional, Tuple
import numpy as np
-from iminuit import Minuit
-
from cal_tools.enums import BadPixelsFF
+from iminuit import Minuit
def any_in(mask: np.ndarray, bits: int) -> bool:
diff --git a/cal_tools/cal_tools/lpdlib.py b/cal_tools/cal_tools/lpdlib.py
index 217c769fcc6faef919697b2b671a1c9e3a65c488..fd0d5132f78eded0210b0df32fca7244fc85a2df 100644
--- a/cal_tools/cal_tools/lpdlib.py
+++ b/cal_tools/cal_tools/lpdlib.py
@@ -3,10 +3,9 @@ from typing import Optional, Tuple
import h5py
import numpy as np
-from iCalibrationDB import Conditions, Constants, Detectors
-
from cal_tools.enums import BadPixels
from cal_tools.tools import get_constant_from_db, get_constant_from_db_and_time
+from iCalibrationDB import Conditions, Constants, Detectors
class LpdCorrections:
diff --git a/cal_tools/cal_tools/plotting.py b/cal_tools/cal_tools/plotting.py
index d85cbd1716a887edcd37f7058915f25538d87f70..63c98c5d561dffc005a0b0d2b35345f3c1b5c0db 100644
--- a/cal_tools/cal_tools/plotting.py
+++ b/cal_tools/cal_tools/plotting.py
@@ -3,8 +3,12 @@ from typing import Any, Dict, Optional
import matplotlib.pyplot as plt
import numpy as np
-from extra_geom import (AGIPD_1MGeometry, AGIPD_500K2GGeometry,
- DSSC_1MGeometry, LPD_1MGeometry)
+from extra_geom import (
+ AGIPD_1MGeometry,
+ AGIPD_500K2GGeometry,
+ DSSC_1MGeometry,
+ LPD_1MGeometry,
+)
from extra_geom import tests as eg_tests
from matplotlib import colors
from matplotlib.patches import Patch
diff --git a/notebooks/AGIPD/AGIPD_Characterize_Gain_Combine_NBC.ipynb b/notebooks/AGIPD/AGIPD_Characterize_Gain_Combine_NBC.ipynb
index 67525d3db1a65ccd8507a2c478863161f3dcdd31..253789c14f642f41c4fe62ef44f8b44533ebf041 100644
--- a/notebooks/AGIPD/AGIPD_Characterize_Gain_Combine_NBC.ipynb
+++ b/notebooks/AGIPD/AGIPD_Characterize_Gain_Combine_NBC.ipynb
@@ -74,9 +74,10 @@
},
"outputs": [],
"source": [
- "from cal_tools.enums import BadPixels\n",
- "from IPython.display import HTML, display, Markdown, Latex\n",
"import tabulate\n",
+ "from cal_tools.enums import BadPixels\n",
+ "from IPython.display import HTML, Latex, Markdown, display\n",
+ "\n",
"table = []\n",
"for item in BadPixels:\n",
" table.append((item.name, \"{:016b}\".format(item.value)))\n",
@@ -104,30 +105,40 @@
},
"outputs": [],
"source": [
+ "import os\n",
+ "\n",
"# std library imports\n",
"from functools import partial\n",
+ "\n",
"import h5py\n",
- "import os\n",
+ "import matplotlib\n",
"\n",
"# numpy and matplot lib specific\n",
"import numpy as np\n",
- "import matplotlib\n",
+ "\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.pyplot as plt\n",
+ "\n",
"%matplotlib inline\n",
"\n",
"import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
- "# pyDetLib imports\n",
- "import XFELDetAna.xfelpycaltools as xcal\n",
- "import XFELDetAna.xfelpyanatools as xana\n",
+ "from datetime import datetime\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "from cal_tools.tools import gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name\n",
+ "import XFELDetAna.xfelpyanatools as xana\n",
"\n",
- "from cal_tools.plotting import show_overview, plot_badpix_3d\n",
- "from datetime import datetime\n",
+ "# pyDetLib imports\n",
+ "import XFELDetAna.xfelpycaltools as xcal\n",
+ "from cal_tools.plotting import plot_badpix_3d, show_overview\n",
+ "from cal_tools.tools import (\n",
+ " gain_map_files,\n",
+ " get_notebook_name,\n",
+ " parse_runs,\n",
+ " run_prop_seq_from_path,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
"\n",
"# usually no need to change these lines\n",
"sensor_size = [128, 512]\n",
@@ -185,8 +196,10 @@
},
"outputs": [],
"source": [
- "from dateutil import parser\n",
"from collections import OrderedDict\n",
+ "\n",
+ "from dateutil import parser\n",
+ "\n",
"offset_g = OrderedDict()\n",
"noise_g = OrderedDict()\n",
"threshold_o = OrderedDict()\n",
@@ -635,7 +648,7 @@
" bp[th[...,1] == 0, 2] |= BadPixels.GAIN_THRESHOLDING_ERROR.value\n",
" \n",
" constants[qm]['BadPixels'] = bp\n",
- " \n"
+ " "
]
},
{
@@ -667,6 +680,7 @@
"import matplotlib.pyplot as plt\n",
"from mpl_toolkits.axes_grid1 import AxesGrid\n",
"\n",
+ "\n",
"def show_overview(cell_to_preview, gain_to_preview):\n",
" \n",
" for module, data in constants.items():\n",
@@ -949,7 +963,7 @@
" bp = data['BadPixels'][...,gain]\n",
" any_bd = np.zeros_like(bp)\n",
" any_bd[bp != 0] = 1\n",
- " plot_badpix_3d(any_bd, cols, title=mod, rebin_fac=rebin, azim=22.5)\n"
+ " plot_badpix_3d(any_bd, cols, title=mod, rebin_fac=rebin, azim=22.5)"
]
},
{
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 926498bd0c239437fbe7df7592337c1e3748af8d..c4cbf1ef2759f6ade29cb6f4861bca470707e085 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -142,14 +142,21 @@
"sns.set_style(\"ticks\")\n",
"\n",
"import seaborn as sns\n",
- "from cal_tools.agipdlib import (AgipdCorrections, get_acq_rate,\n",
- " get_gain_setting, get_num_cells)\n",
+ "from cal_tools.agipdlib import (\n",
+ " AgipdCorrections,\n",
+ " get_acq_rate,\n",
+ " get_gain_setting,\n",
+ " get_num_cells,\n",
+ ")\n",
"from cal_tools.ana_tools import get_range\n",
"from cal_tools.cython import agipdalgs as calgs\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
- "from cal_tools.tools import (CalibrationMetadata, get_dir_creation_date,\n",
- " map_modules_from_folder)\n",
+ "from cal_tools.tools import (\n",
+ " CalibrationMetadata,\n",
+ " get_dir_creation_date,\n",
+ " map_modules_from_folder,\n",
+ ")\n",
"\n",
"sns.set()\n",
"sns.set_context(\"paper\", font_scale=1.4)\n",
diff --git a/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb b/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
index a774ea46a5d6ed6c7b6e93f943f92551d8a7161a..f136170e0a1d7a8965115e60230b410ab14b3392 100644
--- a/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
+++ b/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
@@ -39,7 +39,7 @@
"h_sums_tresh = 100 # Threshold for number of entries in histograms\n",
"\n",
"n_cores_hists = 20 # Number of processes (cores) to create histograms\n",
- "n_cores_files = 10 # Number of processes (cores) to read files\n"
+ "n_cores_files = 10 # Number of processes (cores) to read files"
]
},
{
@@ -49,22 +49,21 @@
"outputs": [],
"source": [
"import glob\n",
- "from multiprocessing import Pipe, Process, Queue, Pool\n",
"import os\n",
"import sys\n",
- "from time import sleep, time, perf_counter\n",
+ "import traceback\n",
"import warnings\n",
+ "from multiprocessing import Pipe, Pool, Process, Queue\n",
+ "from time import perf_counter, sleep, time\n",
"\n",
"import h5py\n",
- "import matplotlib.pyplot as plt\n",
"import matplotlib as mpl\n",
+ "import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import sharedmem\n",
- "import traceback\n",
- "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
- "\n",
- "from cal_tools.cython import agipdalgs as calgs\n",
"from cal_tools.ana_tools import save_dict_to_hdf5\n",
+ "from cal_tools.cython import agipdalgs as calgs\n",
+ "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"\n",
"%matplotlib inline\n",
"warnings.filterwarnings('ignore')"
@@ -129,7 +128,7 @@
" processed[run] = sharedmem.empty((max(run_sequences+[1])+1, max(cells+[1])+1), dtype='i4')\n",
" n_files += len(fnames)\n",
" for file in fnames:\n",
- " file_to_read.put((file, np.array(cells)))\n"
+ " file_to_read.put((file, np.array(cells)))"
]
},
{
@@ -180,7 +179,7 @@
" n_img = len(sel)\n",
" shared_data[i_proc]['data'][:n_img] = group['data'][()][sel]\n",
" shared_data[i_proc]['cellId'][:n_img] = group['cellId'][sel]\n",
- " shared_data[i_proc]['nImg'][:] = n_img\n"
+ " shared_data[i_proc]['nImg'][:] = n_img"
]
},
{
@@ -473,7 +472,7 @@
"hist_std = np.sqrt(np.sum(hist_norm[:, :n_bins-1, ...] * hist_sqr, axis=1))\n",
"\n",
"h_out = (h_sums.astype(np.float32) - total_img[:, None, None].astype(\n",
- " np.float32))/total_img[:, None, None].astype(np.float32)\n"
+ " np.float32))/total_img[:, None, None].astype(np.float32)"
]
},
{
@@ -597,7 +596,7 @@
"ax0.set_xlabel('Cell', fontsize=14)\n",
"ax0.set_ylabel('Average fraction of outliers', fontsize=14)\n",
"ax0.set_title(f'Module {module}', fontsize=16, fontweight='bold')\n",
- "ax0.grid()\n"
+ "ax0.grid()"
]
},
{
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 6ce5f20bfc96f50caf3a6f6dd72a89d38c7bf467..8126a0a01a063d298eb2d950f61978d08f923d4f 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -107,8 +107,11 @@
"\n",
"import matplotlib.pyplot as plt\n",
"from cal_tools.agipdlib import get_gain_setting\n",
- "from cal_tools.tools import (CalibrationMetadata, get_dir_creation_date,\n",
- " map_modules_from_folder)\n",
+ "from cal_tools.tools import (\n",
+ " CalibrationMetadata,\n",
+ " get_dir_creation_date,\n",
+ " map_modules_from_folder,\n",
+ ")\n",
"from dateutil import parser\n",
"from iCalibrationDB import Conditions, Constants, Detectors"
]
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 18d532b1639687b7f0b043d03178befa8748deb0..8d11bae6992695d8f4e4220f9b5b2c7b0a27b8de 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -85,35 +85,50 @@
},
"outputs": [],
"source": [
+ "import warnings\n",
+ "\n",
"# imports and things that do not usually need to be changed\n",
"from datetime import datetime\n",
+ "\n",
"import dateutil.parser\n",
- "import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
- "from collections import OrderedDict\n",
"import os\n",
- "from typing import Tuple, List\n",
+ "from collections import OrderedDict\n",
+ "from typing import List, Tuple\n",
+ "\n",
"import h5py\n",
- "import numpy as np\n",
"import matplotlib\n",
+ "import numpy as np\n",
"import tabulate\n",
"\n",
"matplotlib.use('agg')\n",
"import matplotlib.pyplot as plt\n",
- "from IPython.display import display, Markdown, Latex\n",
+ "from IPython.display import Latex, Markdown, display\n",
+ "\n",
"%matplotlib inline\n",
"\n",
- "from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
- " get_notebook_name, get_pdu_from_db,\n",
- " get_random_db_interface, get_report,\n",
- " map_gain_stages, parse_runs,\n",
- " run_prop_seq_from_path, save_const_to_h5,\n",
- " send_to_db)\n",
- "from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import (create_constant_overview,\n",
- " plot_badpix_3d, show_processed_modules,\n",
- " show_overview)\n",
"from cal_tools.agipdlib import get_bias_voltage, get_gain_setting\n",
+ "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.plotting import (\n",
+ " create_constant_overview,\n",
+ " plot_badpix_3d,\n",
+ " show_overview,\n",
+ " show_processed_modules,\n",
+ ")\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_from_db,\n",
+ " get_notebook_name,\n",
+ " get_pdu_from_db,\n",
+ " get_random_db_interface,\n",
+ " get_report,\n",
+ " map_gain_stages,\n",
+ " parse_runs,\n",
+ " run_prop_seq_from_path,\n",
+ " save_const_to_h5,\n",
+ " send_to_db,\n",
+ ")\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
"from ipyparallel import Client\n",
@@ -121,7 +136,7 @@
"view = Client(profile=cluster_profile)[:]\n",
"view.use_dill()\n",
"\n",
- "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
+ "from iCalibrationDB import Conditions, Constants, Detectors, Versions\n",
"\n",
"gains = np.arange(3)\n",
"\n",
@@ -287,6 +302,8 @@
"source": [
"import copy\n",
"from functools import partial\n",
+ "\n",
+ "\n",
"def characterize_module(il_mode: bool,\n",
" cells: int,\n",
" bp_thresh: Tuple[List[int], float, List[int], float], \n",
@@ -298,11 +315,12 @@
" control_names: List[str],\n",
" karabo_id_control: str,\n",
" inp: Tuple[str, int, int]) -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
- " import numpy as np\n",
" import copy\n",
+ "\n",
" import h5py\n",
+ " import numpy as np\n",
+ " from cal_tools.agipdlib import get_acq_rate, get_num_cells\n",
" from cal_tools.enums import BadPixels\n",
- " from cal_tools.agipdlib import get_num_cells, get_acq_rate\n",
"\n",
" fast_data_filename, channel, gg = inp\n",
" \n",
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
index 695eb6634ff61193d8665a58812667821ad55a6c..9d24310f85738222a28c9592b3d1e21c2803dc3a 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
@@ -76,27 +76,31 @@
"outputs": [],
"source": [
"import glob\n",
- "from multiprocessing import Pool\n",
"import os\n",
"import traceback\n",
"import warnings\n",
+ "from multiprocessing import Pool\n",
"\n",
"import h5py\n",
- "from iminuit import Minuit\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import sharedmem\n",
- "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
- "from XFELDetAna.plotting.simpleplot import simplePlot\n",
"import XFELDetAna.xfelpyanatools as xana\n",
- "\n",
- "from cal_tools.ana_tools import save_dict_to_hdf5, get_range\n",
"from cal_tools.agipdlib import get_bias_voltage\n",
"from cal_tools.agipdutils_ff import (\n",
- " any_in, gaussian, gaussian_sum, get_mask,\n",
- " get_starting_parameters, set_par_limits, fit_n_peaks\n",
+ " any_in,\n",
+ " fit_n_peaks,\n",
+ " gaussian,\n",
+ " gaussian_sum,\n",
+ " get_mask,\n",
+ " get_starting_parameters,\n",
+ " set_par_limits,\n",
")\n",
+ "from cal_tools.ana_tools import get_range, save_dict_to_hdf5\n",
"from cal_tools.enums import BadPixelsFF\n",
+ "from iminuit import Minuit\n",
+ "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
+ "from XFELDetAna.plotting.simpleplot import simplePlot\n",
"\n",
"# %load_ext autotime\n",
"%matplotlib inline\n",
@@ -407,7 +411,7 @@
"gain_mean = np.nanmean(gains, axis=(1,2))\n",
"\n",
"fit_result['mask'][fit_result['gain'] > gain_mean[:,None,None]*gain_lim[1] ] |= BadPixelsFF.GAIN_DEVIATION.value\n",
- "fit_result['mask'][fit_result['gain'] < gain_mean[:,None,None]*gain_lim[0] ] |= BadPixelsFF.GAIN_DEVIATION.value\n"
+ "fit_result['mask'][fit_result['gain'] < gain_mean[:,None,None]*gain_lim[0] ] |= BadPixelsFF.GAIN_DEVIATION.value"
]
},
{
@@ -606,7 +610,7 @@
" ax0.set_xlabel('Memory Cell ID', fontsize=14)\n",
" ax0.set_ylabel(labels[i], fontsize=14)\n",
" ax0.grid()\n",
- " _ = ax0.legend()\n"
+ " _ = ax0.legend()"
]
},
{
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
index da71858d2a7021f315e5692bb4c4a057574a5199..e5c31dd6c69912ad55c9467003b6b799694278df 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
@@ -70,30 +70,39 @@
"outputs": [],
"source": [
"import glob\n",
- "from multiprocessing import Pool\n",
- "import re\n",
"import os\n",
+ "import re\n",
"import traceback\n",
"import warnings\n",
+ "from multiprocessing import Pool\n",
"\n",
- "from dateutil import parser\n",
- "from extra_data import RunDirectory, stack_detector_data\n",
- "from extra_geom import AGIPD_1MGeometry\n",
"import h5py\n",
- "from iCalibrationDB import Detectors, Conditions, Constants\n",
- "from iminuit import Minuit\n",
- "from IPython.display import HTML, display, Markdown, Latex\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import tabulate\n",
- "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
- "from XFELDetAna.plotting.simpleplot import simplePlot\n",
- "\n",
+ "from cal_tools.agipdlib import (\n",
+ " get_acq_rate,\n",
+ " get_bias_voltage,\n",
+ " get_gain_setting,\n",
+ " get_num_cells,\n",
+ ")\n",
+ "from cal_tools.agipdutils_ff import (\n",
+ " any_in,\n",
+ " fit_n_peaks,\n",
+ " gaussian_sum,\n",
+ " get_starting_parameters,\n",
+ ")\n",
"from cal_tools.ana_tools import get_range, save_dict_to_hdf5\n",
- "from cal_tools.agipdlib import get_acq_rate, get_bias_voltage, get_gain_setting, get_num_cells\n",
- "from cal_tools.agipdutils_ff import any_in, fit_n_peaks, gaussian_sum, get_starting_parameters\n",
- "from cal_tools.tools import get_dir_creation_date, send_to_db\n",
"from cal_tools.enums import BadPixels, BadPixelsFF\n",
+ "from cal_tools.tools import get_dir_creation_date, send_to_db\n",
+ "from dateutil import parser\n",
+ "from extra_data import RunDirectory, stack_detector_data\n",
+ "from extra_geom import AGIPD_1MGeometry\n",
+ "from iCalibrationDB import Conditions, Constants, Detectors\n",
+ "from iminuit import Minuit\n",
+ "from IPython.display import HTML, Latex, Markdown, display\n",
+ "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
+ "from XFELDetAna.plotting.simpleplot import simplePlot\n",
"\n",
"%matplotlib inline\n",
"warnings.filterwarnings('ignore')"
@@ -181,7 +190,7 @@
" modules.append(mod)\n",
" except Exception as e:\n",
" err = f\"Error: {e}\\nError traceback: {traceback.format_exc()}\"\n",
- " print(f\"No fit data available for module {qm}\")\n"
+ " print(f\"No fit data available for module {qm}\")"
]
},
{
@@ -331,7 +340,7 @@
"for i in range(16):\n",
" if i in slopesFF:\n",
" const_data['slopesFF'][i,:,pixel_range[0]:pixel_range[2],\n",
- " pixel_range[1]:pixel_range[3]] = slopesFF[i]\n"
+ " pixel_range[1]:pixel_range[3]] = slopesFF[i]"
]
},
{
@@ -479,7 +488,7 @@
"corrections = const_data['slopesFF'] # (16,shape[0],512,128) shape[0]= cell_range[1]-cell_range[0] /\n",
"corrections = np.moveaxis(corrections,1,0) # (shape[0],16,512,128)\n",
"rel_corr = corrections/np.nanmedian(corrections)\n",
- "corrected = orig / rel_corr\n"
+ "corrected = orig / rel_corr"
]
},
{
@@ -574,7 +583,7 @@
"plt.xlabel('[ADU]')\n",
"plt.legend()\n",
"plt.yscale('log')\n",
- "plt.show()\n"
+ "plt.show()"
]
},
{
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index d3d9dbfc349e4114ce4c09772b8f068f8e507940..d8a42e40137e402c4827cc0d5c64edd9991528cb 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -74,29 +74,39 @@
"source": [
"# imports, usually no need to change anything here\n",
"\n",
- "from datetime import datetime\n",
- "from functools import partial\n",
"import os\n",
"import warnings\n",
+ "from datetime import datetime\n",
+ "from functools import partial\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import dateutil.parser\n",
"import h5py\n",
- "from ipyparallel import Client\n",
- "import numpy as np\n",
"import matplotlib\n",
+ "import numpy as np\n",
+ "from ipyparallel import Client\n",
+ "\n",
"matplotlib.use(\"Qt4Agg\")\n",
"import matplotlib.pyplot as plt\n",
+ "\n",
"%matplotlib inline\n",
"\n",
+ "import XFELDetAna.xfelpyanatools as xana\n",
"from cal_tools.agipdlib import get_acq_rate, get_gain_setting, get_num_cells\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import plot_badpix_3d, show_overview\n",
- "from cal_tools.tools import (gain_map_files, get_dir_creation_date, get_notebook_name,\n",
- " parse_runs, run_prop_seq_from_path, get_dir_creation_date,\n",
- " get_pdu_from_db, get_report, send_to_db)\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "import XFELDetAna.xfelpyanatools as xana\n",
+ "from cal_tools.tools import (\n",
+ " gain_map_files,\n",
+ " get_dir_creation_date,\n",
+ " get_notebook_name,\n",
+ " get_pdu_from_db,\n",
+ " get_report,\n",
+ " parse_runs,\n",
+ " run_prop_seq_from_path,\n",
+ " send_to_db,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
"view = Client(profile=cluster_profile)[:]\n",
@@ -246,9 +256,10 @@
"source": [
"def read_and_merge_module_data(cells, path_temp, image_name_temp,\n",
" runs, seqs, il_mode, rawversion, instrument, channel):\n",
+ " import os\n",
+ "\n",
" import h5py\n",
" import numpy as np\n",
- " import os\n",
"\n",
" \n",
" def cal_bursts_per_file(run, dseq=0):\n",
@@ -390,7 +401,7 @@
"p = partial(read_and_merge_module_data, maxcells, path_temp, image_name_temp,\n",
" runs, seqs, IL_MODE, rawversion, instrument)\n",
"# chunk this a bit, so that we don't overuse available memory\n",
- "res = list(map(p, modules))\n"
+ "res = list(map(p, modules))"
]
},
{
@@ -425,9 +436,10 @@
},
"outputs": [],
"source": [
- "from sklearn.cluster import KMeans\n",
"from iminuit import Minuit\n",
- "from iminuit.util import make_func_code, describe\n",
+ "from iminuit.util import describe, make_func_code\n",
+ "from sklearn.cluster import KMeans\n",
+ "\n",
"\n",
"def calc_m_cluster(x, y):\n",
" scan_range = 15\n",
@@ -561,6 +573,7 @@
"outputs": [],
"source": [
"from cal_tools.tools import get_constant_from_db_and_time\n",
+ "\n",
"offsets = {}\n",
"noises = {}\n",
"thresholds = {}\n",
@@ -628,6 +641,7 @@
" test_cells = tcell.tolist()\n",
" \n",
"from mpl_toolkits.axes_grid1 import ImageGrid\n",
+ "\n",
"for mod, r in enumerate(res):\n",
" dig, ana, cellId = r\n",
" d = []\n",
@@ -1025,7 +1039,7 @@
"\n",
" fig = xana.simplePlot(d3, secondpanel=True, y_label=\"PC signal (ADU)\", figsize='2col', aspect=2,\n",
" x_label=\"step #\", y2_label=\"Residuals ($\\sigma$)\", y2_range=(-5,5))\n",
- " fig.savefig(\"{}/module_{}_pixel_plot_fits_fail.png\".format(out_folder, modules[mod]))\n"
+ " fig.savefig(\"{}/module_{}_pixel_plot_fits_fail.png\".format(out_folder, modules[mod]))"
]
},
{
@@ -1043,10 +1057,10 @@
"# Here we perform the calculations in column-parallel for all modules\n",
"def calibrate_single_row(cells, fit_hook, inp):\n",
" \n",
- " from sklearn.cluster import KMeans\n",
- " from iminuit import Minuit\n",
- " from iminuit.util import make_func_code, describe\n",
" import numpy as np\n",
+ " from iminuit import Minuit\n",
+ " from iminuit.util import describe, make_func_code\n",
+ " from sklearn.cluster import KMeans\n",
" \n",
" yrd, yra, offset, noise = inp\n",
"\n",
@@ -1382,6 +1396,7 @@
"outputs": [],
"source": [
"from collections import OrderedDict\n",
+ "\n",
"bad_pixels = OrderedDict()\n",
"for qm, data in fres.items(): \n",
" mask = np.zeros(data['ml'].shape, np.uint32)\n",
@@ -1439,7 +1454,6 @@
},
"outputs": [],
"source": [
- "\n",
"# TODO: add db_module when received from myMDC\n",
"# Create the modules dict of karabo_das and PDUs\n",
"qm_dict = OrderedDict()\n",
@@ -1529,9 +1543,9 @@
},
"outputs": [],
"source": [
+ "import matplotlib.patches as patches\n",
"import matplotlib.pyplot as plt\n",
"from mpl_toolkits.axes_grid1 import AxesGrid\n",
- "import matplotlib.patches as patches\n",
"\n",
"cell_to_preview = min(59, mem_cells-1)\n",
"for module, data in fres.items():\n",
@@ -1626,6 +1640,7 @@
" \"ml\": \"Slope (linear)\",\n",
" \"bl\": \"Offset (linear) (ADU)\"}\n",
"from matplotlib.colors import LogNorm, PowerNorm\n",
+ "\n",
"for module, data in fres.items():\n",
" \n",
" bins = 100\n",
@@ -1709,6 +1724,7 @@
"else:\n",
" test_cells = tcell.tolist()\n",
"from mpl_toolkits.axes_grid1 import ImageGrid\n",
+ "\n",
"for mod, r in enumerate(res):\n",
" dig, ana, cellId = r\n",
" d = []\n",
diff --git a/notebooks/AGIPD/playground/Characterize_AGIPD_Gain_FlatFields_Coupling.ipynb b/notebooks/AGIPD/playground/Characterize_AGIPD_Gain_FlatFields_Coupling.ipynb
index 887a4edbdfb37e6aad04887f195fb9699faa2aa8..738892f6aa07ed9bce39b9cbecd01fbad504e009 100644
--- a/notebooks/AGIPD/playground/Characterize_AGIPD_Gain_FlatFields_Coupling.ipynb
+++ b/notebooks/AGIPD/playground/Characterize_AGIPD_Gain_FlatFields_Coupling.ipynb
@@ -23,7 +23,10 @@
"cell_type": "code",
"execution_count": 1,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -102,7 +105,10 @@
"cell_type": "code",
"execution_count": 2,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [],
"source": [
@@ -136,7 +142,10 @@
"cell_type": "code",
"execution_count": 3,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [],
"source": [
@@ -266,6 +275,9 @@
"execution_count": 29,
"metadata": {
"collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
"scrolled": true
},
"outputs": [
@@ -331,7 +343,10 @@
"cell_type": "code",
"execution_count": 35,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [],
"source": [
@@ -367,7 +382,9 @@
"execution_count": 12,
"metadata": {
"collapsed": false,
- "scrolled": false
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -562,6 +579,9 @@
"execution_count": 37,
"metadata": {
"collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
"scrolled": true
},
"outputs": [
@@ -608,7 +628,9 @@
"execution_count": 14,
"metadata": {
"collapsed": false,
- "scrolled": false
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -817,7 +839,10 @@
"cell_type": "code",
"execution_count": 38,
"metadata": {
- "collapsed": true
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ }
},
"outputs": [],
"source": [
@@ -854,6 +879,9 @@
"execution_count": 21,
"metadata": {
"collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
"scrolled": true
},
"outputs": [
@@ -1049,7 +1077,9 @@
"execution_count": 39,
"metadata": {
"collapsed": false,
- "scrolled": false
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -1084,21 +1114,12 @@
]
},
{
- "cell_type": "code",
- "execution_count": 40,
+ "cell_type": "markdown",
"metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "ename": "SyntaxError",
- "evalue": "invalid syntax (<ipython-input-40-9478b32c4bfd>, line 1)",
- "output_type": "error",
- "traceback": [
- "\u001b[0;36m File \u001b[0;32m\"<ipython-input-40-9478b32c4bfd>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m We plot the distributions of the inidividual pixel coupling for all pixels and memory cells on a given module.\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
- ]
+ "jupyter": {
+ "outputs_hidden": false
}
- ],
+ },
"source": [
"We plot the distributions of the inidividual pixel coupling for all pixels and memory cells on a given module."
]
@@ -1108,7 +1129,9 @@
"execution_count": 23,
"metadata": {
"collapsed": false,
- "scrolled": false
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -1320,9 +1343,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.4.3"
+ "version": "3.6.7"
}
},
"nbformat": 4,
- "nbformat_minor": 1
+ "nbformat_minor": 4
}
diff --git a/notebooks/AGIPD/playground/Investigate_Baseline_Drift.ipynb b/notebooks/AGIPD/playground/Investigate_Baseline_Drift.ipynb
index a013fec502b3094880042dbceaf6d79e9a3bbfec..5d944cca8be5ef31394edfb504c8806297fd3ebc 100644
--- a/notebooks/AGIPD/playground/Investigate_Baseline_Drift.ipynb
+++ b/notebooks/AGIPD/playground/Investigate_Baseline_Drift.ipynb
@@ -4,7 +4,10 @@
"cell_type": "code",
"execution_count": 11,
"metadata": {
- "collapsed": true
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ }
},
"outputs": [],
"source": [
@@ -29,7 +32,10 @@
"cell_type": "code",
"execution_count": 12,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [],
"source": [
@@ -51,7 +57,10 @@
"cell_type": "code",
"execution_count": 13,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -91,7 +100,10 @@
"cell_type": "code",
"execution_count": 14,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -138,7 +150,10 @@
"cell_type": "code",
"execution_count": 15,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -185,7 +200,10 @@
"cell_type": "code",
"execution_count": 16,
"metadata": {
- "collapsed": true
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ }
},
"outputs": [],
"source": [
@@ -198,7 +216,10 @@
"cell_type": "code",
"execution_count": 17,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -228,7 +249,10 @@
"cell_type": "code",
"execution_count": 18,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -253,7 +277,10 @@
"cell_type": "code",
"execution_count": 19,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [],
"source": [
@@ -265,7 +292,10 @@
"cell_type": "code",
"execution_count": 20,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [],
"source": [
@@ -290,7 +320,10 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ }
},
"outputs": [],
"source": [
@@ -301,7 +334,10 @@
"cell_type": "code",
"execution_count": 21,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [],
"source": [
@@ -317,6 +353,9 @@
"execution_count": 193,
"metadata": {
"collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
"scrolled": true
},
"outputs": [],
@@ -471,7 +510,10 @@
"cell_type": "code",
"execution_count": 194,
"metadata": {
- "collapsed": true
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ }
},
"outputs": [],
"source": [
@@ -492,7 +534,10 @@
"cell_type": "code",
"execution_count": 196,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -555,7 +600,10 @@
"cell_type": "code",
"execution_count": 162,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -597,7 +645,9 @@
"execution_count": 36,
"metadata": {
"collapsed": false,
- "scrolled": false
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -712,7 +762,7 @@
" # maximum high gain signal\n",
" dd = copy.copy(d)\n",
" # shift while the medium gain produces negative values on aver\n",
- " def get_neg_med_fraction(pc)\n",
+ " def get_neg_med_fraction(pc):\n",
" np.nanmedian(dd[g==1])\n",
" \n",
"\n",
@@ -786,7 +836,10 @@
"cell_type": "code",
"execution_count": 428,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -830,7 +883,10 @@
"cell_type": "code",
"execution_count": 546,
"metadata": {
- "collapsed": true
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ }
},
"outputs": [],
"source": [
@@ -843,7 +899,10 @@
"cell_type": "code",
"execution_count": 549,
"metadata": {
- "collapsed": false
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ }
},
"outputs": [
{
@@ -1066,7 +1125,10 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ }
},
"outputs": [],
"source": []
@@ -1075,7 +1137,10 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ }
},
"outputs": [],
"source": []
@@ -1097,9 +1162,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.4.3"
+ "version": "3.6.7"
}
},
"nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
}
diff --git a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
index 6ad80ff6010432c439814a1a91905b44fb842682..e3381c13eebacc4b44b644edb19bc1e55e47c847 100644
--- a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
+++ b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
@@ -60,7 +60,7 @@
"instrument = \"SCS\" # the instrument\n",
"high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. Runtime: ~ 1h\n",
"slow_data_aggregators = [1,2,3,4] # quadrant/aggregator\n",
- "operation_mode = '' # Detector operation mode, optional\n"
+ "operation_mode = '' # Detector operation mode, optional"
]
},
{
@@ -74,37 +74,49 @@
},
"outputs": [],
"source": [
- "# imports and things that do not usually need to be changed\n",
- "from datetime import datetime\n",
"import os\n",
"import warnings\n",
+ "\n",
+ "# imports and things that do not usually need to be changed\n",
+ "from datetime import datetime\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"from collections import OrderedDict\n",
"\n",
"import h5py\n",
- "from ipyparallel import Client\n",
- "from IPython.display import display, Markdown, Latex\n",
"import matplotlib\n",
+ "from ipyparallel import Client\n",
+ "from IPython.display import Latex, Markdown, display\n",
+ "\n",
"matplotlib.use('agg')\n",
"import matplotlib.pyplot as plt\n",
+ "\n",
"%matplotlib inline\n",
"import numpy as np\n",
"import tabulate\n",
- "\n",
+ "from cal_tools.dssclib import get_dssc_ctrl_data, get_pulseid_checksum\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import (create_constant_overview,\n",
- " plot_badpix_3d, show_overview,\n",
- " show_processed_modules)\n",
- "from cal_tools.tools import (get_dir_creation_date, get_from_db, \n",
- " get_pdu_from_db, get_notebook_name,\n",
- " get_random_db_interface, get_report,\n",
- " map_gain_stages, parse_runs,\n",
- " run_prop_seq_from_path,\n",
- " save_const_to_h5, send_to_db)\n",
- "from cal_tools.dssclib import (get_dssc_ctrl_data,\n",
- " get_pulseid_checksum)\n",
- "\n",
- "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
+ "from cal_tools.plotting import (\n",
+ " create_constant_overview,\n",
+ " plot_badpix_3d,\n",
+ " show_overview,\n",
+ " show_processed_modules,\n",
+ ")\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_from_db,\n",
+ " get_notebook_name,\n",
+ " get_pdu_from_db,\n",
+ " get_random_db_interface,\n",
+ " get_report,\n",
+ " map_gain_stages,\n",
+ " parse_runs,\n",
+ " run_prop_seq_from_path,\n",
+ " save_const_to_h5,\n",
+ " send_to_db,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants, Detectors, Versions\n",
+ "\n",
"view = Client(profile=cluster_profile)[:]\n",
"view.use_dill()\n",
"\n",
@@ -187,7 +199,7 @@
"os.makedirs(out_folder, exist_ok=True)\n",
"gmf = map_gain_stages(in_folder, offset_runs, path_template, karabo_da, sequences)\n",
"gain_mapped_files, total_sequences, total_file_size = gmf\n",
- "print(f\"Will process a total of {total_sequences} file.\")\n"
+ "print(f\"Will process a total of {total_sequences} file.\")"
]
},
{
@@ -212,10 +224,13 @@
"source": [
"import copy\n",
"from functools import partial\n",
+ "\n",
+ "\n",
"def characterize_module(cells, bp_thresh, rawversion, karabo_id, h5path, h5path_idx, inp):\n",
- " import numpy as np\n",
" import copy\n",
+ "\n",
" import h5py\n",
+ " import numpy as np\n",
" from cal_tools.enums import BadPixels\n",
" \n",
" def get_num_cells(fname, h5path):\n",
@@ -367,7 +382,7 @@
" max_cells = np.max(all_cells)\n",
" print(f\"Using {max_cells} memory cells\")\n",
"else:\n",
- " raise ValueError(\"0 processed memory cells. No raw data available.\")\n"
+ " raise ValueError(\"0 processed memory cells. No raw data available.\")"
]
},
{
@@ -513,7 +528,7 @@
" print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
" f\"• pulseid_checksum: {pidsum}\\n• acquisition_rate: {opfreq}\\n\"\n",
" f\"• target_gain: {targetgain}\\n• encoded_gain: {encodedgain}\\n\"\n",
- " f\"• creation_time: {creation_time}\\n\")\n"
+ " f\"• creation_time: {creation_time}\\n\")"
]
},
{
@@ -528,7 +543,7 @@
" display(Markdown(f'## Position of the module {qm} and its ASICs##'))\n",
" mnames.append(qm)\n",
" \n",
- "show_processed_modules(dinstance=dinstance, constants=None, mnames=mnames, mode=\"position\")\n"
+ "show_processed_modules(dinstance=dinstance, constants=None, mnames=mnames, mode=\"position\")"
]
},
{
diff --git a/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb b/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb
index 75ead05e79776c5489d0d7907799369c6b74c3c6..4d5bc0eb53bac3158b4950f7679f82976016e13c 100644
--- a/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb
+++ b/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb
@@ -71,32 +71,37 @@
},
"outputs": [],
"source": [
+ "# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
+ "import os\n",
"import sys\n",
"from collections import OrderedDict\n",
"\n",
- "# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
- "import os\n",
"import h5py\n",
- "import numpy as np\n",
"import matplotlib\n",
+ "import numpy as np\n",
+ "\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.pyplot as plt\n",
"from ipyparallel import Client\n",
- "from IPython.display import display, Markdown, Latex\n",
+ "from IPython.display import Latex, Markdown, display\n",
+ "\n",
"print(f\"Connecting to profile {cluster_profile}\")\n",
"view = Client(profile=cluster_profile)[:]\n",
"view.use_dill()\n",
"\n",
- "from iCalibrationDB import (ConstantMetaData, Constants, Conditions,\\\n",
- " Detectors, Versions)\n",
- "from cal_tools.tools import (map_modules_from_folder, parse_runs,\\\n",
- " run_prop_seq_from_path, get_notebook_name,\n",
- " get_dir_creation_date, get_constant_from_db)\n",
- "\n",
- "from cal_tools.dssclib import (get_dssc_ctrl_data, get_pulseid_checksum)\n",
- "\n",
+ "from datetime import timedelta\n",
+ "\n",
+ "from cal_tools.dssclib import get_dssc_ctrl_data, get_pulseid_checksum\n",
+ "from cal_tools.tools import (\n",
+ " get_constant_from_db,\n",
+ " get_dir_creation_date,\n",
+ " get_notebook_name,\n",
+ " map_modules_from_folder,\n",
+ " parse_runs,\n",
+ " run_prop_seq_from_path,\n",
+ ")\n",
"from dateutil import parser\n",
- "from datetime import timedelta"
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions"
]
},
{
@@ -139,6 +144,7 @@
" raise AttributeError(\"Output path exists! Exiting\")\n",
"\n",
"import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"print(f\"Detector in use is {karabo_id}\")"
@@ -180,8 +186,10 @@
"outputs": [],
"source": [
"import copy\n",
- "from IPython.display import HTML, display, Markdown, Latex\n",
+ "\n",
"import tabulate\n",
+ "from IPython.display import HTML, Latex, Markdown, display\n",
+ "\n",
"print(f\"Processing a total of {total_sequences} sequence files in chunks of {MAX_PAR}\")\n",
"table = []\n",
"mfc = copy.copy(mapped_files)\n",
@@ -216,21 +224,29 @@
"source": [
"import copy\n",
"from functools import partial\n",
+ "\n",
+ "\n",
"def correct_module(total_sequences, sequences_qm, karabo_id, dinstance, mask_noisy_asic, \n",
" mask_cold_asic, noisy_pix_threshold, chunksize, mem_cells, bias_voltage,\n",
" cal_db_timeout, creation_time, cal_db_interface, h5path, h5path_idx, inp):\n",
" \n",
- " import numpy as np\n",
+ " import binascii\n",
" import copy\n",
+ " import struct\n",
+ " from hashlib import blake2b\n",
+ "\n",
" import h5py\n",
- " from cal_tools.dssclib import (get_dssc_ctrl_data, get_pulseid_checksum)\n",
+ " import numpy as np\n",
+ " from cal_tools.dssclib import get_dssc_ctrl_data, get_pulseid_checksum\n",
" from cal_tools.enums import BadPixels\n",
" from cal_tools.tools import get_constant_from_db_and_time\n",
- " from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- " \n",
- " from hashlib import blake2b\n",
- " import struct\n",
- " import binascii\n",
+ " from iCalibrationDB import (\n",
+ " Conditions,\n",
+ " ConstantMetaData,\n",
+ " Constants,\n",
+ " Detectors,\n",
+ " Versions,\n",
+ " )\n",
" \n",
" filename, filename_out, channel, karabo_da, qm, conditions = inp\n",
" \n",
@@ -488,11 +504,12 @@
},
"outputs": [],
"source": [
- "from mpl_toolkits.mplot3d import Axes3D\n",
"import matplotlib.pyplot as plt\n",
- "from matplotlib import cm\n",
- "from matplotlib.ticker import LinearLocator, FormatStrFormatter\n",
"import numpy as np\n",
+ "from matplotlib import cm\n",
+ "from matplotlib.ticker import FormatStrFormatter, LinearLocator\n",
+ "from mpl_toolkits.mplot3d import Axes3D\n",
+ "\n",
"%matplotlib inline\n",
"def do_3d_plot(data, edges, x_axis, y_axis):\n",
" fig = plt.figure(figsize=(10,10))\n",
@@ -535,7 +552,7 @@
" cb = fig.colorbar(im)\n",
" cb.set_label(\"Counts\")\n",
" \n",
- " \n"
+ " "
]
},
{
@@ -863,9 +880,10 @@
},
"outputs": [],
"source": [
- "from cal_tools.enums import BadPixels\n",
- "from IPython.display import HTML, display, Markdown, Latex\n",
"import tabulate\n",
+ "from cal_tools.enums import BadPixels\n",
+ "from IPython.display import HTML, Latex, Markdown, display\n",
+ "\n",
"table = []\n",
"for item in BadPixels:\n",
" table.append((item.name, \"{:016b}\".format(item.value)))\n",
diff --git a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
index b72c092174d599bfd474bca09b7a3a48c6ebbbb8..a91683d1ba28baf421109730ba4bcc9d60056a00 100644
--- a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
+++ b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
@@ -60,7 +60,7 @@
"ADU_to_electron_upper_mg = 23.4 # for upper hemisphere and medium gain\n",
"ADU_to_electron_lower_mg = 23.4 # for lower hemisphere and medium gain\n",
"ADU_to_electron_upper_lg = 49.3 # for upper hemisphere and low gain\n",
- "ADU_to_electron_lower_lg = 47.3 # for lower hemisphere and low gain\n"
+ "ADU_to_electron_lower_lg = 47.3 # for lower hemisphere and low gain"
]
},
{
@@ -77,30 +77,38 @@
"# Required Packages:\n",
"import copy\n",
"import datetime\n",
- "import time\n",
"import os\n",
+ "import time\n",
"import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import h5py\n",
- "from IPython.display import display, Markdown\n",
"import matplotlib.pyplot as plt\n",
+ "from IPython.display import Markdown, display\n",
+ "\n",
"%matplotlib inline\n",
"import numpy as np\n",
- "from prettytable import PrettyTable\n",
- "\n",
- "from cal_tools.tools import (get_dir_creation_date, get_random_db_interface,\n",
- " get_pdu_from_db, get_report,\n",
- " save_const_to_h5, send_to_db)\n",
"from cal_tools.enums import BadPixels\n",
- "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_pdu_from_db,\n",
+ " get_random_db_interface,\n",
+ " get_report,\n",
+ " save_const_to_h5,\n",
+ " send_to_db,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants, Detectors, Versions\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
+ "from prettytable import PrettyTable\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
"from XFELDetAna.util import env\n",
+ "\n",
"env.iprofile = cluster_profile\n",
"import XFELDetAna.xfelprofiler as xprof\n",
+ "\n",
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
"from XFELDetAna.xfelreaders import ChunkReader"
@@ -689,8 +697,9 @@
"source": [
"def create_circular_mask(h, w, center=None, radius=None):\n",
"\n",
- " import numpy as np\n",
" import math\n",
+ "\n",
+ " import numpy as np\n",
" \n",
" if center is None: # use the middle of the image\n",
" center = [int(w/2), int(h/2)]\n",
diff --git a/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb b/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb
index c879d49e8fb22ba19c94ffc59b1982443f33ddf1..4e08d0f39fddc4c2841445f9c5059621f2ca8159 100644
--- a/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb
+++ b/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb
@@ -67,7 +67,7 @@
"\n",
"def balance_sequences(in_folder, run, sequences, sequences_per_node, karabo_da):\n",
" from xfel_calibrate.calibrate import balance_sequences as bs\n",
- " return bs(in_folder, run, sequences, sequences_per_node, karabo_da)\n"
+ " return bs(in_folder, run, sequences, sequences_per_node, karabo_da)"
]
},
{
@@ -118,37 +118,38 @@
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
"from XFELDetAna.util import env\n",
+ "\n",
"env.iprofile = cluster_profile\n",
"\n",
"import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
- "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna import xfelpyanatools as xana\n",
+ "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna.plotting.util import prettyPlotting\n",
+ "\n",
"prettyPlotting=True\n",
- "from XFELDetAna.xfelreaders import ChunkReader\n",
- "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
+ "import copy\n",
+ "import os\n",
+ "import time\n",
+ "from datetime import timedelta\n",
"\n",
- "import numpy as np\n",
- "import h5py\n",
"import dateutil.parser\n",
+ "import h5py\n",
"import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "from cal_tools.tools import (\n",
+ " get_constant_from_db_and_time,\n",
+ " get_dir_creation_date,\n",
+ " get_random_db_interface,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
+ "from iCalibrationDB.detectors import DetectorTypes\n",
"from iminuit import Minuit\n",
- "\n",
- "import time\n",
- "import copy\n",
- "import os\n",
- "\n",
"from prettytable import PrettyTable\n",
- "\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "from iCalibrationDB.detectors import DetectorTypes\n",
- "from cal_tools.tools import (get_dir_creation_date,\n",
- " get_random_db_interface,\n",
- " get_constant_from_db_and_time)\n",
- "\n",
- "from datetime import timedelta\n",
+ "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
+ "from XFELDetAna.xfelreaders import ChunkReader\n",
"\n",
"%matplotlib inline\n",
"\n",
@@ -156,7 +157,7 @@
" sequences = None\n",
"\n",
"# select a random port for the data base \n",
- "cal_db_interface = get_random_db_interface(cal_db_interface)\n"
+ "cal_db_interface = get_random_db_interface(cal_db_interface)"
]
},
{
@@ -193,12 +194,12 @@
"print(\"Data is output to: {}\".format(out_folder))\n",
"\n",
"import datetime\n",
+ "\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run) + timedelta(days=time_offset_days)\n",
"if creation_time:\n",
- " print(\"Using {} as creation time\".format(creation_time.isoformat()))\n",
- "\n"
+ " print(\"Using {} as creation time\".format(creation_time.isoformat()))"
]
},
{
@@ -212,7 +213,6 @@
},
"outputs": [],
"source": [
- "\n",
"sensorSize = [x, y]\n",
"chunkSize = 100 #Number of images to read per chunk\n",
"blockSize = [sensorSize[0]//2, sensorSize[1]] #Sensor area will be analysed according to blocksize\n",
@@ -245,7 +245,7 @@
" os.makedirs(out_folder)\n",
"elif not overwrite:\n",
" # Stop Notebook not only this cell\n",
- " raise SystemExit(\"Output path exists! Exiting\") \n"
+ " raise SystemExit(\"Output path exists! Exiting\") "
]
},
{
@@ -298,8 +298,10 @@
"outputs": [],
"source": [
"import copy\n",
- "from IPython.display import HTML, display, Markdown, Latex\n",
+ "\n",
"import tabulate\n",
+ "from IPython.display import HTML, Latex, Markdown, display\n",
+ "\n",
"print(\"Processing a total of {} sequence files\".format(total_sequences))\n",
"table = []\n",
"\n",
@@ -549,7 +551,7 @@
" flip_rgain = creation_time.replace(tzinfo=None) >= flipped_between[0] and creation_time.replace(tzinfo=None) <= flipped_between[1]\n",
" flip_rgain &= (relgain_time.replace(tzinfo=None) >= flipped_between[0] \n",
" and relgain_time.replace(tzinfo=None) <= flipped_between[1])\n",
- " print(\"Accounting for flipped detector: {}\".format(flip_rgain))\n"
+ " print(\"Accounting for flipped detector: {}\".format(flip_rgain))"
]
},
{
@@ -600,9 +602,7 @@
" blockSize=[x//2, y],\n",
" runParallel=True)\n",
"\n",
- " \n",
- "\n",
- "\n"
+ " "
]
},
{
@@ -913,7 +913,7 @@
" single_im_cc = data[...,0]\n",
" \n",
" except Exception as e:\n",
- " print(\"Couldn't calibrate data in {}: {}\".format(f, e))\n"
+ " print(\"Couldn't calibrate data in {}: {}\".format(f, e))"
]
},
{
@@ -1047,7 +1047,7 @@
" pmap = ax.imshow(patternStatsUH[m][j], interpolation=\"nearest\", vmax=np.median(patternStatsUH[m][j]))\n",
" ax.set_title(m+\"(\"+str(j)+\")\")\n",
" cb = fig.colorbar(pmap)\n",
- " k+=1\n"
+ " k+=1"
]
},
{
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index f1c6c165d58328b8be8e5fbc215825c9d6309f52..cd968ed1f16d1109ae4175e64337aa8d30bcf9f0 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -73,8 +73,11 @@
"import numpy as np\n",
"import tabulate\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.tools import (get_constant_from_db_and_time,\n",
- " get_dir_creation_date, map_modules_from_folder)\n",
+ "from cal_tools.tools import (\n",
+ " get_constant_from_db_and_time,\n",
+ " get_dir_creation_date,\n",
+ " map_modules_from_folder,\n",
+ ")\n",
"from iCalibrationDB import Conditions, Constants\n",
"from IPython.display import Latex, display\n",
"from matplotlib.colors import LogNorm\n",
diff --git a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
index a0d0bfef4937f2d075385a66f2acb824924fac50..5edd0344aa075d478e9c6820f92a66df4fe1b624 100644
--- a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
@@ -67,29 +67,40 @@
"import glob\n",
"import os\n",
"import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import h5py\n",
- "from h5py import File as h5file\n",
"import matplotlib\n",
+ "from h5py import File as h5file\n",
+ "\n",
"matplotlib.use('agg')\n",
"import matplotlib.pyplot as plt\n",
+ "\n",
"%matplotlib inline\n",
"import numpy as np\n",
- "\n",
- "from cal_tools.enums import BadPixels\n",
- "from cal_tools.tools import (get_dir_creation_date, get_pdu_from_db,\n",
- " get_random_db_interface, get_report,\n",
- " save_const_to_h5, send_to_db)\n",
"from cal_tools.ana_tools import save_dict_to_hdf5\n",
- "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
- "\n",
+ "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_pdu_from_db,\n",
+ " get_random_db_interface,\n",
+ " get_report,\n",
+ " save_const_to_h5,\n",
+ " send_to_db,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants, Detectors, Versions\n",
"from XFELDetAna.util import env\n",
+ "\n",
"env.iprofile = cluster_profile\n",
- "from XFELDetAna.detectors.jungfrau import readerPSI as jfreaderPSI\n",
"from XFELDetAna.detectors.jungfrau import reader as jfreader\n",
+ "from XFELDetAna.detectors.jungfrau import readerPSI as jfreaderPSI\n",
"from XFELDetAna.detectors.jungfrau.jf_chunk_reader import JFChunkReader\n",
- "from XFELDetAna.detectors.jungfrau.util import count_n_files, rollout_data, sanitize_data_cellid\n",
+ "from XFELDetAna.detectors.jungfrau.util import (\n",
+ " count_n_files,\n",
+ " rollout_data,\n",
+ " sanitize_data_cellid,\n",
+ ")\n",
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"from XFELDetAna.plotting.histogram import histPlot"
]
@@ -282,8 +293,7 @@
" print('Lost {:d} images out of {:d}'.format(n_empty_sc, this_run_mcells * (n_tr - n_empty_trains)))\n",
"\n",
" else:\n",
- " print('missing G{:01d}'.format(i))\n",
- "\n"
+ " print('missing G{:01d}'.format(i))"
]
},
{
@@ -303,11 +313,12 @@
"source": [
"import matplotlib.pyplot as plt\n",
"from XFELDetAna.core.util import remove_nans\n",
+ "\n",
"%matplotlib inline\n",
"#%matplotlib notebook\n",
"\n",
- "from XFELDetAna.plotting.histogram import histPlot\n",
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
+ "from XFELDetAna.plotting.histogram import histPlot\n",
"\n",
"g_name = ['G0', 'G1', 'G2']\n",
"g_range = [(0, 8000), (8000, 16000), (8000, 16000)]\n",
@@ -324,7 +335,6 @@
},
"outputs": [],
"source": [
- "\n",
"for g_idx in gains:\n",
" for cell in range(0, memoryCells):\n",
" f_o0 = heatmapPlot(np.swapaxes(offset_map[..., cell, g_idx], 0, 1), \n",
diff --git a/notebooks/LPD/Characterize_LPD_GAIN_CI_per_pixel_NBC.ipynb b/notebooks/LPD/Characterize_LPD_GAIN_CI_per_pixel_NBC.ipynb
index b8b17cf511c4601f63021f7c42d40e743969a1bc..e52ca2afaefe83ddadd28dfd177a3111c82b071e 100644
--- a/notebooks/LPD/Characterize_LPD_GAIN_CI_per_pixel_NBC.ipynb
+++ b/notebooks/LPD/Characterize_LPD_GAIN_CI_per_pixel_NBC.ipynb
@@ -55,36 +55,49 @@
},
"outputs": [],
"source": [
+ "import os\n",
+ "import warnings\n",
+ "\n",
"# std library imports\n",
"from functools import partial\n",
+ "\n",
"import h5py\n",
- "import os\n",
- "import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
+ "import matplotlib\n",
+ "\n",
"# numpy and matplot lib specific\n",
"import numpy as np\n",
- "import matplotlib\n",
+ "\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.pyplot as plt\n",
+ "\n",
"%matplotlib inline\n",
"\n",
"# parallel processing via ipcluster\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
"from ipyparallel import Client\n",
+ "\n",
"view = Client(profile=cluster_profile)[:]\n",
"view.use_dill()\n",
"\n",
"from datetime import datetime\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "from cal_tools.tools import gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name, get_dir_creation_date\n",
- "from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
+ "import XFELDetAna.xfelpyanatools as xana\n",
"\n",
"# pyDetLib imports\n",
"import XFELDetAna.xfelpycaltools as xcal\n",
- "import XFELDetAna.xfelpyanatools as xana\n",
+ "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.plotting import create_constant_overview, plot_badpix_3d, show_overview\n",
+ "from cal_tools.tools import (\n",
+ " gain_map_files,\n",
+ " get_dir_creation_date,\n",
+ " get_notebook_name,\n",
+ " parse_runs,\n",
+ " run_prop_seq_from_path,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
"\n",
"# usually no need to change these lines\n",
"sensor_size = [256, 256]\n",
@@ -136,7 +149,6 @@
},
"outputs": [],
"source": [
- "\n",
"offset_g = {}\n",
"noise_g = {}\n",
"if not db_input:\n",
@@ -313,9 +325,9 @@
" \"\"\"\n",
" channel, offset, mod_corr = inp\n",
" \n",
- " import XFELDetAna.xfelpycaltools as xcal\n",
- " import numpy as np\n",
" import h5py\n",
+ " import numpy as np\n",
+ " import XFELDetAna.xfelpycaltools as xcal\n",
"\n",
" def splitOffGainLPD(d):\n",
" msk = np.zeros(d.shape, np.uint16)\n",
@@ -405,6 +417,7 @@
"outputs": [],
"source": [
"from collections import OrderedDict\n",
+ "\n",
"res_uncorr = OrderedDict()\n",
"indices_in_settings = {}\n",
"\n",
@@ -527,7 +540,7 @@
"sort_low = np.argsort(ref_pk_pos_low)\n",
"sort_med = np.argsort(ref_pk_pos_med)\n",
"sort_high = np.argsort(ref_pk_pos_high)\n",
- "ref_pos = ref_pk_pos_low[sort_low], ref_pk_pos_med[sort_med], ref_pk_pos_high[sort_high]\n"
+ "ref_pos = ref_pk_pos_low[sort_low], ref_pk_pos_med[sort_med], ref_pk_pos_high[sort_high]"
]
},
{
@@ -560,7 +573,7 @@
" #idx2 = sort_low[indices_in_settings[\"low\"].index(idx2)]\n",
" slope = np.mean(ref_pk_pos_med[idx1]/ref_pk_pos_low[idx2])\n",
" slopes.append(slope)\n",
- "slope_10_1 = np.mean(slopes)\n"
+ "slope_10_1 = np.mean(slopes)"
]
},
{
@@ -579,7 +592,6 @@
},
"outputs": [],
"source": [
- "\n",
"cell = 1\n",
"for qm, r in res_uncorr.items():\n",
" i = list(modules)[ii]\n",
@@ -916,9 +928,9 @@
" def calibrate_single_row(cells, inp):\n",
"\n",
" \n",
- " from iminuit import Minuit\n",
- " from iminuit.util import make_func_code, describe\n",
" import numpy as np\n",
+ " from iminuit import Minuit\n",
+ " from iminuit.util import describe, make_func_code\n",
"\n",
" yrd, xrd = inp\n",
"\n",
@@ -1242,7 +1254,7 @@
},
"outputs": [],
"source": [
- "mask_low = plot_for_gain(fres_low)\n"
+ "mask_low = plot_for_gain(fres_low)"
]
},
{
@@ -1260,7 +1272,7 @@
},
"outputs": [],
"source": [
- "mask_med = plot_for_gain(fres_med)\n"
+ "mask_med = plot_for_gain(fres_med)"
]
},
{
@@ -1324,7 +1336,6 @@
},
"outputs": [],
"source": [
- "\n",
"ofile = \"{}/lpd_ci_store_{}_16_{}.h5\".format(out_folder,\n",
" \"_\".join([str(m) for m in modules]),\n",
" capacitance)\n",
@@ -1487,9 +1498,9 @@
" return [], [], []\n",
" channel, om, mod_corr, rg, rbg = inp\n",
" \n",
- " import XFELDetAna.xfelpycaltools as xcal\n",
- " import numpy as np\n",
" import h5py\n",
+ " import numpy as np\n",
+ " import XFELDetAna.xfelpycaltools as xcal\n",
"\n",
" def splitOffGainLPD(d):\n",
" msk = np.zeros(d.shape, np.uint16)\n",
@@ -1577,7 +1588,6 @@
},
"outputs": [],
"source": [
- "\n",
"cell = 1\n",
"for i, r in enumerate(res_corr):\n",
" means_low, means_med, means_high = r\n",
diff --git a/notebooks/LPD/LPDChar_Darks_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
index d9c9fa6dddbc646fb9202a3c4dcf4f2ccd651e70..cf833d99f362bfe84f22e780b505deae9b85cedc 100644
--- a/notebooks/LPD/LPDChar_Darks_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
@@ -66,37 +66,49 @@
"metadata": {},
"outputs": [],
"source": [
- "from collections import OrderedDict\n",
"import copy\n",
- "from datetime import datetime\n",
- "from functools import partial\n",
"import os\n",
"import warnings\n",
+ "from collections import OrderedDict\n",
+ "from datetime import datetime\n",
+ "from functools import partial\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import dateutil.parser\n",
"import h5py\n",
- "from ipyparallel import Client\n",
- "from IPython.display import display, Markdown, Latex\n",
"import matplotlib\n",
+ "from ipyparallel import Client\n",
+ "from IPython.display import Latex, Markdown, display\n",
+ "\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.patches as patches\n",
"import matplotlib.pyplot as plt\n",
+ "\n",
"%matplotlib inline\n",
"import numpy as np\n",
"import tabulate\n",
- "\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import (show_overview, plot_badpix_3d, \n",
- " create_constant_overview,\n",
- " show_processed_modules)\n",
- "from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
- " get_notebook_name, get_pdu_from_db,\n",
- " get_random_db_interface, get_report,\n",
- " map_gain_stages, parse_runs, \n",
- " run_prop_seq_from_path,\n",
- " save_const_to_h5, send_to_db)\n",
- "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
+ "from cal_tools.plotting import (\n",
+ " create_constant_overview,\n",
+ " plot_badpix_3d,\n",
+ " show_overview,\n",
+ " show_processed_modules,\n",
+ ")\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_from_db,\n",
+ " get_notebook_name,\n",
+ " get_pdu_from_db,\n",
+ " get_random_db_interface,\n",
+ " get_report,\n",
+ " map_gain_stages,\n",
+ " parse_runs,\n",
+ " run_prop_seq_from_path,\n",
+ " save_const_to_h5,\n",
+ " send_to_db,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants, Detectors, Versions\n",
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"from XFELDetAna.plotting.simpleplot import simplePlot"
]
@@ -187,11 +199,12 @@
"# the actual characterization\n",
"def characterize_module(cells, bp_thresh, skip_first_ntrains, ntrains, test_for_normality, \n",
" h5path, h5path_idx, inp):\n",
- " import numpy as np\n",
" import copy\n",
+ "\n",
" import h5py\n",
- " from cal_tools.enums import BadPixels\n",
+ " import numpy as np\n",
" import scipy.stats\n",
+ " from cal_tools.enums import BadPixels\n",
"\n",
" def splitOffGainLPD(d):\n",
" msk = np.zeros(d.shape, np.uint16)\n",
@@ -641,7 +654,7 @@
" use_axis=ax,\n",
" y_log=False, legend='outside-top-ncol3-frame', legend_pad=0.05, legend_size='5%')\n",
" ax.ticklabel_format(style='sci', axis='y', scilimits=(4,6))\n",
- " \n"
+ " "
]
},
{
@@ -901,7 +914,7 @@
" cmap='viridis', # cb_loc='right',cb_aspect=15,\n",
" x_ticklabels=np.arange(16)+1,\n",
" x_ticks=np.arange(16)+0.5)\n",
- " plt.show()\n"
+ " plt.show()"
]
},
{
@@ -1163,7 +1176,6 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
"header = ['Parameter', \n",
" \"New constant\", \"Old constant \", \n",
" \"New constant\", \"Old constant \", \n",
diff --git a/notebooks/LPD/LPD_Correct_and_Verify.ipynb b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
index 07ba55d4a3d3396b3a1145f2cab11104e52ebfe9..d1fb2336cd3ad84be3af1e2898172e900c9ca276 100644
--- a/notebooks/LPD/LPD_Correct_and_Verify.ipynb
+++ b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
@@ -64,7 +64,7 @@
"\n",
"def balance_sequences(in_folder, run, sequences, sequences_per_node, karabo_da):\n",
" from xfel_calibrate.calibrate import balance_sequences as bs\n",
- " return bs(in_folder, run, sequences, sequences_per_node, karabo_da)\n"
+ " return bs(in_folder, run, sequences, sequences_per_node, karabo_da)"
]
},
{
@@ -79,8 +79,9 @@
"outputs": [],
"source": [
"import sys\n",
- "from datetime import datetime\n",
"import warnings\n",
+ "from datetime import datetime\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"max_cells = mem_cells\n",
@@ -107,24 +108,30 @@
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
"import os\n",
+ "\n",
"import h5py\n",
- "import numpy as np\n",
"import matplotlib\n",
+ "import numpy as np\n",
+ "\n",
"matplotlib.use(\"agg\")\n",
- "import matplotlib.pyplot as plt\n",
- "from ipyparallel import Client\n",
+ "from collections import OrderedDict\n",
+ "from datetime import datetime\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "from cal_tools.tools import (gain_map_files, parse_runs, run_prop_seq_from_path, \n",
- " get_notebook_name, get_dir_creation_date, \n",
- " get_constant_from_db, map_modules_from_folder)\n",
+ "import matplotlib.pyplot as plt\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
"from cal_tools.lpdlib import LpdCorrections\n",
- "from datetime import datetime\n",
- "\n",
- "from collections import OrderedDict\n",
- "from datetime import datetime\n",
+ "from cal_tools.plotting import create_constant_overview, plot_badpix_3d, show_overview\n",
+ "from cal_tools.tools import (\n",
+ " gain_map_files,\n",
+ " get_constant_from_db,\n",
+ " get_dir_creation_date,\n",
+ " get_notebook_name,\n",
+ " map_modules_from_folder,\n",
+ " parse_runs,\n",
+ " run_prop_seq_from_path,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
+ "from ipyparallel import Client\n",
"\n",
"print(\"Connecting to profile {}\".format(cluster_profile))\n",
"view = Client(profile=cluster_profile)[:]\n",
@@ -193,8 +200,10 @@
"outputs": [],
"source": [
"import copy\n",
- "from IPython.display import HTML, display, Markdown, Latex\n",
+ "\n",
"import tabulate\n",
+ "from IPython.display import HTML, Latex, Markdown, display\n",
+ "\n",
"print(\"Processing a total of {} sequence files in chunks of {}\".format(total_sequences, MAX_PAR))\n",
"table = []\n",
"mfc = copy.copy(mapped_files)\n",
@@ -228,17 +237,20 @@
"source": [
"import copy\n",
"from functools import partial\n",
+ "\n",
+ "\n",
"def correct_module(max_cells, do_ff, index_v, CHUNK_SIZE, total_sequences, sequences_qm, \n",
" bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range, max_pulses,\n",
" dbparms, fileparms, nodb, no_non_linear_corrections, mark_non_lin_region, linear_between,\n",
" nlc_version, h5path, h5path_idx, karabo_id, inp):\n",
- " import numpy as np\n",
" import copy\n",
- " import h5py\n",
+ " import os\n",
+ " import re\n",
" import socket\n",
" from datetime import datetime\n",
- " import re\n",
- " import os\n",
+ "\n",
+ " import h5py\n",
+ " import numpy as np\n",
" from cal_tools.enums import BadPixels\n",
" from cal_tools.lpdlib import LpdCorrections\n",
"\n",
@@ -379,7 +391,6 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
"print(\"Offset was injected on: \")\n",
"for k, v in whens.items():\n",
" if v['err'] is None:\n",
@@ -399,11 +410,12 @@
},
"outputs": [],
"source": [
- "from mpl_toolkits.mplot3d import Axes3D\n",
"import matplotlib.pyplot as plt\n",
- "from matplotlib import cm\n",
- "from matplotlib.ticker import LinearLocator, FormatStrFormatter\n",
"import numpy as np\n",
+ "from matplotlib import cm\n",
+ "from matplotlib.ticker import FormatStrFormatter, LinearLocator\n",
+ "from mpl_toolkits.mplot3d import Axes3D\n",
+ "\n",
"%matplotlib inline\n",
"def do_3d_plot(data, edges, x_axis, y_axis):\n",
" fig = plt.figure(figsize=(10,10))\n",
@@ -526,6 +538,7 @@
"d_quads = [(-19+dc[0],-300+dc[1]),(10+dc[0],-9+dc[1]),(-256+dc[0],19+dc[1]),(-285+dc[0],-271+dc[1])] # MAY 2019\n",
"\n",
"import cal_tools.metrology as metro\n",
+ "\n",
"in_files = \"{}/CORR*LPD*S{:05d}*.h5\".format(out_folder, sequences[0] if sequences else 0)\n",
"datapath = \"{}/data\".format(h5path)\n",
"print(\"Preview is from {}\".format(in_files))"
@@ -664,6 +677,7 @@
"# We obtain a histogram for each module, according to its position defined\n",
"# in the coord_list.\n",
"from scipy.stats import binned_statistic_2d\n",
+ "\n",
"hs = []\n",
"bins_nums = []\n",
"edges = []\n",
diff --git a/notebooks/LPD/LPD_FlatField_Radial_per_pixel_CI_NBC.ipynb b/notebooks/LPD/LPD_FlatField_Radial_per_pixel_CI_NBC.ipynb
index 5eeebf141753a5147c9ff5650073b77a4100e9d1..33522e672c31ec05bb170c2e27b21e4f7514140f 100644
--- a/notebooks/LPD/LPD_FlatField_Radial_per_pixel_CI_NBC.ipynb
+++ b/notebooks/LPD/LPD_FlatField_Radial_per_pixel_CI_NBC.ipynb
@@ -52,31 +52,39 @@
},
"outputs": [],
"source": [
- "\n",
"import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
"\n",
"import os\n",
+ "\n",
"import h5py\n",
- "import numpy as np\n",
"import matplotlib\n",
+ "import numpy as np\n",
+ "\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.pyplot as plt\n",
+ "\n",
"%matplotlib inline\n",
"\n",
"\n",
+ "from collections import OrderedDict\n",
"from datetime import datetime\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "from cal_tools.tools import (gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name,\n",
- " get_dir_creation_date, get_constant_from_db)\n",
- "from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
"import cal_tools.metrology as metro\n",
- "\n",
- "from collections import OrderedDict\n",
+ "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.plotting import create_constant_overview, plot_badpix_3d, show_overview\n",
+ "from cal_tools.tools import (\n",
+ " gain_map_files,\n",
+ " get_constant_from_db,\n",
+ " get_dir_creation_date,\n",
+ " get_notebook_name,\n",
+ " parse_runs,\n",
+ " run_prop_seq_from_path,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
"\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
@@ -122,6 +130,7 @@
"source": [
"# set everything up filewise\n",
"from queue import Queue\n",
+ "\n",
"if not os.path.exists(out_folder):\n",
" os.makedirs(out_folder)\n",
" \n",
@@ -155,7 +164,7 @@
" if \"{:05d}.h5\".format(seq) in abs_entry:\n",
" file_list.append(os.path.abspath(abs_entry))\n",
" \n",
- "mapped_files, mod_ids = map_modules_from_files(file_list)\n"
+ "mapped_files, mod_ids = map_modules_from_files(file_list)"
]
},
{
@@ -167,8 +176,9 @@
"outputs": [],
"source": [
"import copy\n",
- "from IPython.display import HTML, display, Markdown, Latex\n",
+ "\n",
"import tabulate\n",
+ "from IPython.display import HTML, Latex, Markdown, display\n",
"\n",
"table = []\n",
"mfc = copy.copy(mapped_files)\n",
@@ -206,7 +216,7 @@
" continue\n",
" fout = os.path.abspath(\"{}/{}\".format(in_folder, (os.path.split(fname_in)[-1]).replace(\"RAW\", \"CORR\")))\n",
" \n",
- " first_files.append((fname_in, fout))\n"
+ " first_files.append((fname_in, fout))"
]
},
{
@@ -269,6 +279,7 @@
"outputs": [],
"source": [
"import cal_tools.metrology as metro\n",
+ "\n",
"in_files = \"{}/CORR*S{:05d}*.h5\".format(in_folder, sequences[0] if sequences else 0)\n",
"datapath = \"INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/data\"\n",
"posarr = metro.positionFileList(in_files, datapath, geometry_file, d_quads, nImages = 100)"
@@ -397,7 +408,6 @@
},
"outputs": [],
"source": [
- "\n",
"positions = []\n",
"mn_tiles = []\n",
"gn_tiles = []\n",
@@ -426,6 +436,7 @@
"source": [
"from matplotlib.colors import LogNorm, PowerNorm\n",
"\n",
+ "\n",
"def translateToModuleBL(tilePositions):\n",
" tileHeight = 17.7 # mm\n",
" # The module origin is the top left corner of the\n",
@@ -540,7 +551,7 @@
" \n",
" \n",
" \n",
- " \n"
+ " "
]
},
{
@@ -589,6 +600,7 @@
"# We obtain a histogram for each module, according to its position defined\n",
"# in the coord_list.\n",
"from scipy.stats import binned_statistic_2d\n",
+ "\n",
"hs = []\n",
"bins_nums = []\n",
"edges = []\n",
diff --git a/notebooks/LPD/Mine_RadIntensity_vs_XGM_NBC.ipynb b/notebooks/LPD/Mine_RadIntensity_vs_XGM_NBC.ipynb
index ce9c4e4d4b6ba70c5d9ff88d55fd0fd582717795..b2e4b1ff9ed05f556ef7e07b6dba55e66f6a487c 100644
--- a/notebooks/LPD/Mine_RadIntensity_vs_XGM_NBC.ipynb
+++ b/notebooks/LPD/Mine_RadIntensity_vs_XGM_NBC.ipynb
@@ -39,7 +39,7 @@
"runs = \"69-72\"\n",
"method = \"average\" # method to use for evaluation of images: radial, average\n",
"\n",
- "sequences = [-1] # range allowed\n"
+ "sequences = [-1] # range allowed"
]
},
{
@@ -51,8 +51,9 @@
"outputs": [],
"source": [
"def create_run_list(proposal_folder, file_prefix, xgm_folder, xgm_prefix, runs):\n",
- " from cal_tools.tools import parse_runs\n",
" import glob\n",
+ "\n",
+ " from cal_tools.tools import parse_runs\n",
" in_runs = parse_runs(runs, return_type=int)\n",
" runs_to_process = []\n",
" for r in in_runs:\n",
@@ -80,17 +81,22 @@
"outputs": [],
"source": [
"import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"import os\n",
+ "\n",
"import h5py\n",
- "import numpy as np\n",
"import matplotlib\n",
+ "import numpy as np\n",
+ "\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.pyplot as plt\n",
+ "\n",
"%matplotlib inline\n",
"from functools import partial\n",
"\n",
"from ipyparallel import Client\n",
+ "\n",
"print(\"Connecting to profile {}\".format(cluster_profile))\n",
"view = Client(profile=cluster_profile)[:]\n",
"view.use_dill()\n",
@@ -115,6 +121,7 @@
"def get_sequences(in_folder, run, sequences):\n",
" import glob\n",
" import re\n",
+ "\n",
" import numpy as np\n",
" \n",
" sequence_files = glob.glob(\"{}/r{:04d}/*-S*.h5\".format(in_folder, run))\n",
@@ -140,11 +147,11 @@
" file_prefix, xgm_prefix, data_path, xgm_path, xgm_path_td, raw_prefix, \n",
" geometry_file, d_quads, temp_out_folder, gain_path, \n",
" adc_folder, adc_prefix, adc_root, adc_paths, method, sequence):\n",
+ " import cal_tools.metrology as metro\n",
" import h5py\n",
" import numpy as np\n",
- " import cal_tools.metrology as metro\n",
" from scipy.stats import binned_statistic_2d\n",
- " \n",
+ "\n",
" # convert the Carthesian coordinates of the detector to polar coordinates\n",
" def mod_cart_to_pol(d, dx, dy, filter_by_val=True):\n",
" \"\"\" Convert carthesian coords to polar coords\n",
diff --git a/notebooks/Tutorial/calversion.ipynb b/notebooks/Tutorial/calversion.ipynb
index 80feabac1bdf3489370e8f53453464ad1caf9744..ca40a9d242ce398911e759990d6a410b1fe90039 100644
--- a/notebooks/Tutorial/calversion.ipynb
+++ b/notebooks/Tutorial/calversion.ipynb
@@ -69,10 +69,11 @@
"outputs": [],
"source": [
"import matplotlib\n",
+ "\n",
"%matplotlib inline\n",
"\n",
- "import numpy as np\n",
"import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
"\n",
"# if not using slurm: make sure a cluster is running with \n",
"# ipcluster start --n=4 --profile=tutorial\n",
@@ -195,6 +196,7 @@
"source": [
"from functools import partial\n",
"\n",
+ "\n",
"def parallel_stats(input):\n",
" return input.mean(), input.std()\n",
"\n",
@@ -202,8 +204,7 @@
"results = view.map_sync(p, fake_data)\n",
"\n",
"p_mean= [ x[0] for x in results ]\n",
- "p_std= [ x[1] for x in results ]\n",
- "\n"
+ "p_std= [ x[1] for x in results ]"
]
},
{
diff --git a/notebooks/Tutorial/startversion.ipynb b/notebooks/Tutorial/startversion.ipynb
index 1a5e4042b4eda3bc0aac57bf262752934b2b1b12..0f3d05fc05b24eaa82d8aa9fc638b9efdddf690b 100644
--- a/notebooks/Tutorial/startversion.ipynb
+++ b/notebooks/Tutorial/startversion.ipynb
@@ -16,10 +16,11 @@
"outputs": [],
"source": [
"import matplotlib\n",
+ "\n",
"%matplotlib inline\n",
"\n",
- "import numpy as np\n",
- "import matplotlib.pyplot as plt"
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np"
]
},
{
diff --git a/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb b/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
index 8270ef2e5deb6c40bcc55ad394c2a9f0b28e0773..89ea1f460e2b16e56cbe860f328686722c901af2 100644
--- a/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
@@ -45,7 +45,7 @@
"bias_voltage = 200 # bias voltage\n",
"in_vacuum = False # detector operated in vacuum\n",
"fix_temperature = 290. # fix temperature to this value\n",
- "operation_mode = '' # Detector operation mode, optional\n"
+ "operation_mode = '' # Detector operation mode, optional"
]
},
{
@@ -56,27 +56,36 @@
"source": [
"import os\n",
"import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import h5py\n",
- "from IPython.display import display, Markdown, Latex\n",
"import matplotlib.pyplot as plt\n",
+ "from IPython.display import Latex, Markdown, display\n",
+ "\n",
"%matplotlib inline\n",
"import numpy as np\n",
- "\n",
- "from cal_tools.tools import (get_dir_creation_date, get_pdu_from_db,\n",
- " get_random_db_interface, get_report,\n",
- " save_const_to_h5, send_to_db)\n",
- "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_pdu_from_db,\n",
+ " get_random_db_interface,\n",
+ " get_report,\n",
+ " save_const_to_h5,\n",
+ " send_to_db,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants, Detectors, Versions\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
"from XFELDetAna.util import env\n",
+ "\n",
"env.iprofile = cluster_profile\n",
- "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna import xfelpyanatools as xana\n",
+ "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
"from XFELDetAna.plotting.util import prettyPlotting\n",
+ "\n",
"prettyPlotting = True\n",
"import XFELDetAna.xfelprofiler as xprof\n",
+ "\n",
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
"from XFELDetAna.xfelreaders import ChunkReader\n",
@@ -164,7 +173,7 @@
" nImages, chunkSize,\n",
" path=h5path,\n",
" pixels_x=sensorSize[0],\n",
- " pixels_y=sensorSize[1], )\n"
+ " pixels_y=sensorSize[1], )"
]
},
{
@@ -179,7 +188,7 @@
"histCalRaw = xcal.HistogramCalculator(sensorSize, bins=1000,\n",
" range=[0, 10000], parallel=False,\n",
" memoryCells=memoryCells,\n",
- " cores=cpuCores, blockSize=blockSize)\n"
+ " cores=cpuCores, blockSize=blockSize)"
]
},
{
@@ -199,7 +208,7 @@
"constant_maps['Noise'] = noiseCal.get() #Produce noise map\n",
"\n",
"noiseCal.reset() #Reset noise calculator\n",
- "print(\"Initial maps were created\")\n"
+ "print(\"Initial maps were created\")"
]
},
{
@@ -249,7 +258,7 @@
" x_label='Columns', y_label='Rows',\n",
" lut_label='Noise (ADU)',\n",
" x_range=(0, y),\n",
- " y_range=(0, x), vmax=2 * np.mean(constant_maps['Noise']))\n"
+ " y_range=(0, x), vmax=2 * np.mean(constant_maps['Noise']))"
]
},
{
diff --git a/notebooks/ePix100/Correction_ePix100_NBC.ipynb b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
index 2c4fd8da983e9f9f7476a63995bba52221066a8b..083e048249c03e15a6ea821b4e0601766dcc361a 100644
--- a/notebooks/ePix100/Correction_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
@@ -58,7 +58,7 @@
" \n",
"def balance_sequences(in_folder, run, sequences, sequences_per_node, karabo_da):\n",
" from xfel_calibrate.calibrate import balance_sequences as bs\n",
- " return bs(in_folder, run, sequences, sequences_per_node, karabo_da)\n"
+ " return bs(in_folder, run, sequences, sequences_per_node, karabo_da)"
]
},
{
@@ -77,27 +77,29 @@
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
"from XFELDetAna.util import env\n",
+ "\n",
"env.iprofile = cluster_profile\n",
"\n",
"import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
- "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna import xfelpyanatools as xana\n",
+ "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna.plotting.util import prettyPlotting\n",
- "prettyPlotting=True\n",
- "from XFELDetAna.xfelreaders import ChunkReader\n",
- "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
"\n",
- "import numpy as np\n",
- "import h5py\n",
- "import time\n",
+ "prettyPlotting=True\n",
"import copy\n",
"import os\n",
+ "import time\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
+ "import h5py\n",
+ "import numpy as np\n",
+ "from cal_tools.tools import get_constant_from_db, get_dir_creation_date\n",
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
- "from cal_tools.tools import get_dir_creation_date, get_constant_from_db\n",
+ "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
+ "from XFELDetAna.xfelreaders import ChunkReader\n",
"\n",
"%matplotlib inline\n",
"\n",
@@ -139,6 +141,7 @@
"print(\"Data is output to: {}\".format(out_folder))\n",
"\n",
"import datetime\n",
+ "\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run)\n",
@@ -182,7 +185,7 @@
"if not os.path.exists(out_folder):\n",
" os.makedirs(out_folder)\n",
"elif not overwrite:\n",
- " raise AttributeError(\"Output path exists! Exiting\") \n"
+ " raise AttributeError(\"Output path exists! Exiting\") "
]
},
{
@@ -234,8 +237,9 @@
},
"outputs": [],
"source": [
- "from IPython.display import HTML, display, Markdown, Latex\n",
"import tabulate\n",
+ "from IPython.display import HTML, Latex, Markdown, display\n",
+ "\n",
"print(\"Processing a total of {} sequence files\".format(total_sequences))\n",
"table = []\n",
"\n",
@@ -336,7 +340,7 @@
" gainMap = np.ones(sensorSize, np.float32)\n",
"\n",
"else:\n",
- " gainMap = np.ones(sensorSize, np.float32)\n"
+ " gainMap = np.ones(sensorSize, np.float32)"
]
},
{
@@ -366,8 +370,7 @@
" parallel=run_parallel,\n",
" cores=cpuCores,\n",
" blockSize=blockSize,\n",
- " gains=None)\n",
- "\n"
+ " gains=None)"
]
},
{
@@ -388,7 +391,7 @@
" range=[-50, 1000], parallel=run_parallel,\n",
" nCells=memoryCells, \n",
" cores=cpuCores,\n",
- " blockSize=blockSize)\n"
+ " blockSize=blockSize)"
]
},
{
@@ -457,7 +460,7 @@
" range=[-50, 1000], parallel=run_parallel,\n",
" nCells=memoryCells, \n",
" cores=cpuCores,\n",
- " blockSize=blockSize)\n"
+ " blockSize=blockSize)"
]
},
{
diff --git a/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb b/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
index 9023752ac105dd3f9b985e7c638edf9f586aebbe..7929498d2fdbe58afcb6d873e810cbae36e30c97 100644
--- a/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
+++ b/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
@@ -45,7 +45,7 @@
"bias_voltage = 200 # bias voltage\n",
"in_vacuum = False # detector operated in vacuum\n",
"fix_temperature = 290. # fix temperature to this value\n",
- "operation_mode = '' # Detector operation mode, optional\n"
+ "operation_mode = '' # Detector operation mode, optional"
]
},
{
@@ -55,27 +55,36 @@
"outputs": [],
"source": [
"import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"import os\n",
"\n",
"import h5py\n",
"import matplotlib.pyplot as plt\n",
+ "\n",
"%matplotlib inline\n",
"import numpy as np\n",
- "\n",
- "from cal_tools.tools import (get_dir_creation_date, get_pdu_from_db,\n",
- " get_random_db_interface, get_report,\n",
- " save_const_to_h5, send_to_db)\n",
- "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_pdu_from_db,\n",
+ " get_random_db_interface,\n",
+ " get_report,\n",
+ " save_const_to_h5,\n",
+ " send_to_db,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants, Detectors, Versions\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna.plotting.util import prettyPlotting\n",
+ "\n",
"prettyPlotting = True\n",
"from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
"from XFELDetAna.util import env\n",
+ "\n",
"env.iprofile = cluster_profile\n",
"import XFELDetAna.xfelprofiler as xprof\n",
+ "\n",
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
"from XFELDetAna.xfelreaders import ChunkReader\n",
@@ -88,7 +97,7 @@
" if limit == 0:\n",
" return nImages\n",
" else:\n",
- " return min(nImages, limit)\n"
+ " return min(nImages, limit)"
]
},
{
@@ -115,7 +124,7 @@
"print(\"HDF5 path: {}\".format(h5path))\n",
"if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run)\n",
- " print(\"Using {} as creation time\".format(creation_time.isoformat()))\n"
+ " print(\"Using {} as creation time\".format(creation_time.isoformat()))"
]
},
{
@@ -167,7 +176,7 @@
" nImages, chunkSize,\n",
" path=h5path,\n",
" pixels_x=sensorSize[0],\n",
- " pixels_y=sensorSize[1], )\n"
+ " pixels_y=sensorSize[1], )"
]
},
{
@@ -182,7 +191,7 @@
"histCalRaw = xcal.HistogramCalculator(sensorSize, bins=1000,\n",
" range=[0, 10000], parallel=False,\n",
" memoryCells=memoryCells,\n",
- " cores=cpuCores, blockSize=blockSize)\n"
+ " cores=cpuCores, blockSize=blockSize)"
]
},
{
@@ -203,7 +212,7 @@
"constant_maps['Noise'] = noiseCal.get() #Produce noise map\n",
"\n",
"noiseCal.reset() #Reset noise calculator\n",
- "print(\"Initial maps were created\")\n"
+ "print(\"Initial maps were created\")"
]
},
{
@@ -253,7 +262,7 @@
" x_label='Columns', y_label='Rows',\n",
" lut_label='Noise (ADU)',\n",
" x_range=(0, y),\n",
- " y_range=(0, x), vmax=2 * np.mean(constant_maps['Noise']))\n"
+ " y_range=(0, x), vmax=2 * np.mean(constant_maps['Noise']))"
]
},
{
diff --git a/notebooks/ePix10K/Correction_ePix10K_NBC.ipynb b/notebooks/ePix10K/Correction_ePix10K_NBC.ipynb
index 53646280d7b2bee429531cdff5be19a3ce6213fe..a85e3ae3afca1bb4193deaa4178c8612a001dedf 100644
--- a/notebooks/ePix10K/Correction_ePix10K_NBC.ipynb
+++ b/notebooks/ePix10K/Correction_ePix10K_NBC.ipynb
@@ -74,27 +74,29 @@
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
"from XFELDetAna.util import env\n",
+ "\n",
"env.iprofile = cluster_profile\n",
"\n",
"import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
- "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna import xfelpyanatools as xana\n",
+ "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna.plotting.util import prettyPlotting\n",
- "prettyPlotting=True\n",
- "from XFELDetAna.xfelreaders import ChunkReader\n",
- "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
"\n",
- "import numpy as np\n",
- "import h5py\n",
- "import time\n",
+ "prettyPlotting=True\n",
"import copy\n",
"import os\n",
+ "import time\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
+ "import h5py\n",
+ "import numpy as np\n",
+ "from cal_tools.tools import get_constant_from_db, get_dir_creation_date\n",
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
- "from cal_tools.tools import get_dir_creation_date, get_constant_from_db\n",
+ "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
+ "from XFELDetAna.xfelreaders import ChunkReader\n",
"\n",
"%matplotlib inline\n",
"\n",
@@ -130,6 +132,7 @@
"print(\"Data is output to: {}\".format(out_folder))\n",
"\n",
"import datetime\n",
+ "\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run)\n",
@@ -175,7 +178,7 @@
"if not os.path.exists(out_folder):\n",
" os.makedirs(out_folder)\n",
"elif not overwrite:\n",
- " raise AttributeError(\"Output path exists! Exiting\") \n"
+ " raise AttributeError(\"Output path exists! Exiting\") "
]
},
{
@@ -227,8 +230,9 @@
},
"outputs": [],
"source": [
- "from IPython.display import HTML, display, Markdown, Latex\n",
"import tabulate\n",
+ "from IPython.display import HTML, Latex, Markdown, display\n",
+ "\n",
"print(\"Processing a total of {} sequence files\".format(total_sequences))\n",
"table = []\n",
"\n",
@@ -328,7 +332,7 @@
" nCells = memoryCells, \n",
" cores=cpuCores, gains=None,\n",
" blockSize=blockSize,\n",
- " parallel=run_parallel)\n"
+ " parallel=run_parallel)"
]
},
{
@@ -349,7 +353,7 @@
" range=[-50, 1000], parallel=run_parallel,\n",
" nCells=memoryCells, \n",
" cores=cpuCores,\n",
- " blockSize=blockSize)\n"
+ " blockSize=blockSize)"
]
},
{
@@ -417,7 +421,7 @@
" range=[-50, 1000], parallel=run_parallel,\n",
" nCells=memoryCells, \n",
" cores=cpuCores,\n",
- " blockSize=blockSize)\n"
+ " blockSize=blockSize)"
]
},
{
diff --git a/notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb b/notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb
index c6f0b34783e6a50460ec8d8a6b4dda1f1c327cd3..78522d66048689388a5e99d544f8873986d8df06 100644
--- a/notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb
+++ b/notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb
@@ -55,14 +55,16 @@
"source": [
"import copy\n",
"import datetime\n",
- "import h5py\n",
"import inspect\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
+ "import h5py\n",
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
+ "\n",
"\n",
"def extend_parms(detector_instance):\n",
- " from iCalibrationDB import Conditions\n",
" import inspect\n",
+ "\n",
+ " from iCalibrationDB import Conditions\n",
" existing = set()\n",
" def extract_parms(cls):\n",
" args, varargs, varkw, defaults = inspect.getargspec(cls.__init__)\n",
diff --git a/notebooks/generic/Interpolate_Constants.ipynb b/notebooks/generic/Interpolate_Constants.ipynb
index a1fa814b03eb1b3303540e16c2f7feb2251c3ef7..d3c46c223ddbffb6ab6d5fffe28f6e84be5b16e6 100644
--- a/notebooks/generic/Interpolate_Constants.ipynb
+++ b/notebooks/generic/Interpolate_Constants.ipynb
@@ -65,18 +65,22 @@
"source": [
"import copy\n",
"import datetime\n",
- "import h5py\n",
"import inspect\n",
+ "\n",
+ "import h5py\n",
+ "import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"from cal_tools.enums import BadPixels\n",
- "import matplotlib.pyplot as plt\n",
+ "\n",
"%matplotlib inline\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
+ "\n",
"\n",
"def extend_parms(detector_instance):\n",
- " from iCalibrationDB import Conditions\n",
" import inspect\n",
+ "\n",
+ " from iCalibrationDB import Conditions\n",
" existing = set()\n",
" def extract_parms(cls):\n",
" args, varargs, varkw, defaults = inspect.getargspec(cls.__init__)\n",
diff --git a/notebooks/generic/Interpolate_Constants_NBC.ipynb b/notebooks/generic/Interpolate_Constants_NBC.ipynb
index 7ac1b8c4ddd0753c01550b0337d8232d51d400bb..203a7a8528d8504051690ff5fce73da81a6405fa 100644
--- a/notebooks/generic/Interpolate_Constants_NBC.ipynb
+++ b/notebooks/generic/Interpolate_Constants_NBC.ipynb
@@ -61,16 +61,18 @@
"source": [
"import copy\n",
"import datetime\n",
- "import h5py\n",
"import inspect\n",
+ "\n",
+ "import h5py\n",
"import numpy as np\n",
"from cal_tools.enums import BadPixels\n",
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
"\n",
"def extend_parms(detector_instance):\n",
- " from iCalibrationDB import Conditions\n",
" import inspect\n",
+ "\n",
+ " from iCalibrationDB import Conditions\n",
" existing = set()\n",
" def extract_parms(cls):\n",
" args, varargs, varkw, defaults = inspect.getargspec(cls.__init__)\n",
diff --git a/notebooks/generic/PlotFromCalDB_NBC.ipynb b/notebooks/generic/PlotFromCalDB_NBC.ipynb
index 8240ddf2f6d95173151f0fec4d283d5da27e9cde..45c18e7b2a52bca0b14243eabd4be0d92b6c8548 100644
--- a/notebooks/generic/PlotFromCalDB_NBC.ipynb
+++ b/notebooks/generic/PlotFromCalDB_NBC.ipynb
@@ -66,20 +66,28 @@
"source": [
"import copy\n",
"import datetime\n",
- "import dateutil.parser\n",
- "import numpy as np\n",
"import os\n",
"import sys\n",
"import warnings\n",
+ "\n",
+ "import dateutil.parser\n",
+ "import numpy as np\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import matplotlib.pyplot as plt\n",
- "\n",
- "from iCalibrationDB import Constants, Conditions, Detectors, ConstantMetaData\n",
+ "from cal_tools.ana_tools import (\n",
+ " HMType,\n",
+ " IMType,\n",
+ " combine_constants,\n",
+ " combine_lists,\n",
+ " get_range,\n",
+ " hm_combine,\n",
+ " load_data_from_hdf5,\n",
+ " save_dict_to_hdf5,\n",
+ ")\n",
"from cal_tools.tools import get_from_db, get_random_db_interface\n",
- "from cal_tools.ana_tools import (save_dict_to_hdf5, load_data_from_hdf5, \n",
- " HMType, IMType, hm_combine, combine_constants,\n",
- " combine_lists, get_range)"
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors"
]
},
{
@@ -656,7 +664,7 @@
" 'MeanBP': ['dataBP', 'Good pixels only', 'Mean over pixels'],\n",
" 'NBP': ['nBP', '', 'Fraction of BP'],\n",
" 'stdBP': ['dataBPStd', 'Good pixels only', '$\\sigma$ over pixels'],\n",
- "}\n"
+ "}"
]
},
{
@@ -847,7 +855,7 @@
" plt.ylabel('{}, {} {}'.format(const[0], keys[key][2], unit)) \n",
" plt.legend(loc='best guess')\n",
" plt.title(title)\n",
- " fig.savefig(fname)\n"
+ " fig.savefig(fname)"
]
}
],
diff --git a/notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb b/notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb
index 8d6d6e884b1bfe342e963782e9a51f30feaabe7a..21215acfbe46ca594a091b1793fea7fd21e756a4 100644
--- a/notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb
+++ b/notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb
@@ -48,12 +48,17 @@
"outputs": [],
"source": [
"import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"import numpy as np\n",
- "\n",
- "from cal_tools.ana_tools import (load_data_from_hdf5, \n",
- " HMType, IMType, multi_union,\n",
- " hm_combine, get_range)"
+ "from cal_tools.ana_tools import (\n",
+ " HMType,\n",
+ " IMType,\n",
+ " get_range,\n",
+ " hm_combine,\n",
+ " load_data_from_hdf5,\n",
+ " multi_union,\n",
+ ")"
]
},
{
@@ -110,7 +115,7 @@
" 'stdBP': ['dataBPStd', 'Good pixels only', '$\\sigma$ over pixels'],\n",
" 'stdASIC': ['', '', '$\\sigma$ over ASICs'],\n",
" 'stdCell': ['', '', '$\\sigma$ over Cells'],\n",
- "}\n"
+ "}"
]
},
{
@@ -246,7 +251,7 @@
" fname='{}/{}_all_g{}_{}.png'.format(\n",
" out_folder, const[0], gain, key),\n",
" vmin=vmin, vmax=vmax,\n",
- " pad=[0.125, 0.151, 0.12, 0.17], htype=htype)\n"
+ " pad=[0.125, 0.151, 0.12, 0.17], htype=htype)"
]
}
],
diff --git a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
index d82f8d52bdf243615f014900b63b22c1ba6597b3..d64e6f485787f0be3cd0f28d9e9b119e97a2dc73 100644
--- a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
+++ b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
@@ -21,27 +21,31 @@
"metadata": {},
"outputs": [],
"source": [
- "from collections import OrderedDict\n",
"import copy\n",
- "from datetime import datetime\n",
"import os\n",
"import warnings\n",
+ "from collections import OrderedDict\n",
+ "from datetime import datetime\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import glob\n",
+ "\n",
"import h5py\n",
- "from IPython.display import display, Markdown, Latex\n",
- "import numpy as np\n",
"import matplotlib\n",
+ "import numpy as np\n",
+ "from IPython.display import Latex, Markdown, display\n",
+ "\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.gridspec as gridspec\n",
"import matplotlib.patches as patches\n",
"import matplotlib.pyplot as plt\n",
+ "\n",
"%matplotlib inline\n",
+ "import extra_geom\n",
"import tabulate\n",
"from cal_tools.ana_tools import get_range\n",
"from cal_tools.plotting import show_processed_modules\n",
- "import extra_geom\n",
"from iCalibrationDB import Detectors\n",
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"from XFELDetAna.plotting.simpleplot import simplePlot"
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
index d1d59f69a31ad596e68135872f2d1f96964701a0..b23ef32bfcfde18bba57315e5f93fe120e7b07df 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
@@ -70,7 +70,7 @@
"\n",
"run_parallel = True # for parallel computation\n",
"cpuCores = 40 # specifies the number of running cpu cores\n",
- "operation_mode = '' # Detector operation mode, optional\n"
+ "operation_mode = '' # Detector operation mode, optional"
]
},
{
@@ -84,37 +84,46 @@
},
"outputs": [],
"source": [
- "import os\n",
"import copy\n",
"import datetime\n",
+ "import os\n",
"import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import h5py\n",
- "import numpy as np\n",
"import matplotlib.pyplot as plt\n",
- "%matplotlib inline\n",
- "from prettytable import PrettyTable\n",
- "from IPython.display import display, Markdown\n",
+ "import numpy as np\n",
"\n",
+ "%matplotlib inline\n",
+ "import XFELDetAna.xfelprofiler as xprof\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.pnccdlib import extract_slow_data, VALID_GAINS\n",
- "from cal_tools.tools import (get_dir_creation_date, get_pdu_from_db,\n",
- " get_random_db_interface, get_report, \n",
- " save_const_to_h5, send_to_db)\n",
- "from iCalibrationDB import (Constants, Conditions, Detectors, Versions)\n",
+ "from cal_tools.pnccdlib import VALID_GAINS, extract_slow_data\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_pdu_from_db,\n",
+ " get_random_db_interface,\n",
+ " get_report,\n",
+ " save_const_to_h5,\n",
+ " send_to_db,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants, Detectors, Versions\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
- "import XFELDetAna.xfelprofiler as xprof\n",
+ "from IPython.display import Markdown, display\n",
+ "from prettytable import PrettyTable\n",
+ "\n",
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
"from XFELDetAna.util import env\n",
+ "\n",
"env.iprofile = cluster_profile\n",
- "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna import xfelpyanatools as xana\n",
+ "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna.plotting.util import prettyPlotting\n",
+ "\n",
"prettyPlotting=True\n",
- "from XFELDetAna.xfelreaders import ChunkReader\n",
- "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5"
+ "from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
+ "from XFELDetAna.xfelreaders import ChunkReader"
]
},
{
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Gain.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Gain.ipynb
index f76e5d7058da86af23a9e7c8adcfbace38f5e238..1c300edaf15236abf7d7579d0eef23c4038e3ee1 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Gain.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Gain.ipynb
@@ -81,39 +81,48 @@
"source": [
"import copy\n",
"import datetime\n",
- "from datetime import timedelta\n",
"import glob\n",
"import os\n",
"import traceback\n",
"import warnings\n",
+ "from datetime import timedelta\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"from functools import partial\n",
+ "\n",
"import h5py\n",
"import iminuit as im\n",
- "from iminuit import Minuit\n",
- "from IPython.display import display, Markdown\n",
"import matplotlib\n",
+ "from iminuit import Minuit\n",
+ "from IPython.display import Markdown, display\n",
+ "\n",
"%matplotlib inline\n",
"import matplotlib.pyplot as plt\n",
- "from mpl_toolkits.axes_grid1 import ImageGrid, AxesGrid\n",
"import numpy as np\n",
+ "import XFELDetAna.xfelprofiler as xprof\n",
+ "from cal_tools.pnccdlib import VALID_GAINS, extract_slow_data\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_pdu_from_db,\n",
+ " get_report,\n",
+ " save_const_to_h5,\n",
+ " send_to_db,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
+ "from iCalibrationDB.detectors import DetectorTypes\n",
+ "from mpl_toolkits.axes_grid1 import AxesGrid, ImageGrid\n",
"from prettytable import PrettyTable\n",
"\n",
- "from cal_tools.pnccdlib import extract_slow_data, VALID_GAINS\n",
- "from cal_tools.tools import (get_dir_creation_date, get_pdu_from_db,\n",
- " get_report, save_const_to_h5, send_to_db)\n",
- "from iCalibrationDB import (Conditions, ConstantMetaData,\n",
- " Constants, Detectors, Versions)\n",
- "from iCalibrationDB.detectors import DetectorTypes\n",
- "import XFELDetAna.xfelprofiler as xprof\n",
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
"from XFELDetAna.util import env\n",
+ "\n",
"env.iprofile = cluster_profile\n",
- "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna import xfelpyanatools as xana\n",
+ "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna.plotting.util import prettyPlotting\n",
+ "\n",
"prettyPlotting=True\n",
"\n",
"if sequences[0] == -1:\n",
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index 1f206f73f61493e04264ed614faf9befa0d32d4e..9bbaf925b071be253560c81f4ebae5ebce3b87cb 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -105,38 +105,43 @@
"source": [
"import copy\n",
"import datetime\n",
- "from datetime import timedelta\n",
"import glob\n",
"import os\n",
"import time\n",
"import traceback\n",
- "from typing import Tuple\n",
"import warnings\n",
+ "from datetime import timedelta\n",
+ "from typing import Tuple\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import h5py\n",
- "from IPython.display import display, Markdown\n",
- "from iminuit import Minuit\n",
"import matplotlib.pyplot as plt\n",
+ "from iminuit import Minuit\n",
+ "from IPython.display import Markdown, display\n",
+ "\n",
"%matplotlib inline\n",
"import numpy as np\n",
+ "import XFELDetAna.xfelprofiler as xprof\n",
+ "from cal_tools.pnccdlib import VALID_GAINS, extract_slow_data\n",
+ "from cal_tools.tools import (\n",
+ " get_constant_from_db_and_time,\n",
+ " get_dir_creation_date,\n",
+ " get_random_db_interface,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
+ "from iCalibrationDB.detectors import DetectorTypes\n",
"from prettytable import PrettyTable\n",
"\n",
- "from cal_tools.pnccdlib import extract_slow_data, VALID_GAINS\n",
- "from cal_tools.tools import (get_constant_from_db_and_time,\n",
- " get_dir_creation_date,\n",
- " get_random_db_interface)\n",
- "from iCalibrationDB import (Conditions, ConstantMetaData,\n",
- " Constants, Detectors, Versions)\n",
- "from iCalibrationDB.detectors import DetectorTypes\n",
- "import XFELDetAna.xfelprofiler as xprof\n",
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
"from XFELDetAna.util import env\n",
+ "\n",
"env.iprofile = cluster_profile\n",
- "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna import xfelpyanatools as xana\n",
+ "from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna.plotting.util import prettyPlotting\n",
+ "\n",
"prettyPlotting=True\n",
"\n",
"\n",
diff --git a/pyproject.toml b/pyproject.toml
index dc67ef6f76d0b6b251eba4eeed85b7b7e2c8da05..caca0f78278c771d34a230c10378041a35644366 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,2 +1,5 @@
[build-system]
requires = ["cython==0.29.21", "numpy==1.19.1", "setuptools>=40.8.0", "wheel"]
+
+[tool.isort]
+profile = "black"
diff --git a/tests/test_webservice.py b/tests/test_webservice.py
index ff368afe247b878a0e61cafac1c746beae829848..604668a93b4db1365ff1c0773866451bf122373e 100644
--- a/tests/test_webservice.py
+++ b/tests/test_webservice.py
@@ -6,8 +6,7 @@ import pytest
from testpath import MockCommand
sys.path.insert(0, Path(__file__).parent / 'webservice')
-from webservice.webservice import (check_files, merge, parse_config,
- wait_on_transfer)
+from webservice.webservice import check_files, merge, parse_config, wait_on_transfer
def test_check_files():
diff --git a/webservice/serve_overview.py b/webservice/serve_overview.py
index b8c6c5593a8a385a9ed044d9d85031e822f18f4b..a741c6a2071ddee6245b9e3b94044caa95688d31 100644
--- a/webservice/serve_overview.py
+++ b/webservice/serve_overview.py
@@ -11,8 +11,7 @@ from uuid import uuid4
import yaml
from jinja2 import Template
-from xfel_calibrate.settings import (free_nodes_cmd, preempt_nodes_cmd,
- reservation)
+from xfel_calibrate.settings import free_nodes_cmd, preempt_nodes_cmd, reservation
class LimitedSizeDict(OrderedDict):
diff --git a/webservice/webservice.py b/webservice/webservice.py
index 43b0ecc6e6145361dca6d1d767d8135de1a8ec6d..e07b8a42890d1f884682bb61dae5865bea51e793 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -14,7 +14,7 @@ import urllib.parse
from asyncio import get_event_loop, shield
from datetime import datetime
from pathlib import Path
-from subprocess import run, PIPE
+from subprocess import PIPE, run
from threading import Thread
from typing import Any, Dict, List, Optional
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index c85c1e6b9092ed9d309af83308bd6f2a9a764d22..004193424919dc4339e2fe425a41750b67a3f190 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -20,8 +20,7 @@ import nbconvert
import nbformat
import numpy as np
from jinja2 import Template
-from nbparameterise import (extract_parameters, parameter_values,
- replace_definitions)
+from nbparameterise import extract_parameters, parameter_values, replace_definitions
from .finalize import tex_escape
from .notebooks import notebooks