From 77b9d2f272a394cca74e7db8dbf7df707b8d4aaa Mon Sep 17 00:00:00 2001
From: Karim Ahmed <karim.ahmed@xfel.eu>
Date: Tue, 22 Sep 2020 09:37:22 +0200
Subject: [PATCH] [AGIPD] [SlopesPC] use send_to_db
---
cal_tools/cal_tools/tools.py | 6 +-
.../AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb | 102 ++++++++----------
requirements.txt | 2 +-
xfel_calibrate/finalize.py | 2 +-
4 files changed, 46 insertions(+), 66 deletions(-)
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index d6364ba6b..b8f695160 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -6,7 +6,6 @@ from os import environ, listdir, path, stat
from os.path import isfile, splitext
from queue import Queue
import re
-import textwrap
from time import sleep
from urllib.parse import urljoin
@@ -481,13 +480,12 @@ def send_to_db(device, constant, condition, file_loc,
if ntries == 0 and doraise:
raise
except Exception as e:
- if "has already been take" in str(e):
+ if "has already been taken" in str(e):
print(f"WARNING: {constant.name} has already been "
"injected with the same parameter "
"conditions\n")
else:
- # reduce error text if it is too long.
- print("\n".join(textwrap.wrap(str(e), 100)))
+ print(f"{e}\n")
if 'missing_token' in str(e):
ntries -= 1
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index 2542e9d50..bd919cef0 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -37,7 +37,7 @@
"outputs": [],
"source": [
"in_folder = '/gpfs/exfel/exp/SPB/202030/p900138/raw/' # path to input data, required\n",
- "modules = [14,] # modules to work on, required, range allowed\n",
+ "modules = [-1] # modules to work on, required, range allowed\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/pc\" # path to output to, required\n",
"runs = [466, 467, 468, 469, 470, 471, 472, 473] # runs to use, required, range allowed\n",
"n_sequences = 3 # number of sequence files, starting for 0 to evaluate\n",
@@ -72,33 +72,34 @@
"outputs": [],
"source": [
"# imports, usually no need to change anything here\n",
+ "\n",
"from datetime import datetime\n",
- "import dateutil.parser\n",
- "import h5py\n",
+ "from functools import partial\n",
"import os\n",
+ "import warnings\n",
+ "warnings.filterwarnings('ignore')\n",
"\n",
+ "import dateutil.parser\n",
+ "import h5py\n",
+ "from ipyparallel import Client\n",
"import numpy as np\n",
"import matplotlib\n",
"matplotlib.use(\"Qt4Agg\")\n",
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
- "# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
- "from ipyparallel import Client\n",
"\n",
- "view = Client(profile=cluster_profile)[:]\n",
- "view.use_dill()\n",
- "\n",
- "from functools import partial\n",
+ "from cal_tools.agipdlib import get_acq_rate, get_gain_setting, get_num_cells\n",
+ "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.influx import InfluxLogger\n",
+ "from cal_tools.plotting import plot_badpix_3d, show_overview\n",
+ "from cal_tools.tools import (gain_map_files, get_dir_creation_date, get_notebook_name,\n",
+ " parse_runs, run_prop_seq_from_path, get_dir_creation_date, send_to_db)\n",
+ "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
"import XFELDetAna.xfelpyanatools as xana\n",
- "import warnings\n",
- "warnings.filterwarnings('ignore')\n",
"\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "from cal_tools.tools import gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name, get_dir_creation_date\n",
- "from cal_tools.influx import InfluxLogger\n",
- "from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import show_overview, plot_badpix_3d\n",
- "from cal_tools.agipdlib import get_acq_rate, get_num_cells, get_gain_setting\n",
+ "# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
+ "view = Client(profile=cluster_profile)[:]\n",
+ "view.use_dill()\n",
"\n",
"IL_MODE = interlaced \n",
"maxcells = mem_cells if not interlaced else mem_cells*2\n",
@@ -1436,53 +1437,34 @@
},
"outputs": [],
"source": [
+ "md = None\n",
+ "\n",
"if db_output:\n",
+ "\n",
+ " det = getattr(Detectors, dinstance)\n",
+ "\n",
" for qm, r in fres.items():\n",
- " metadata = ConstantMetaData()\n",
- " slopespc = Constants.AGIPD.SlopesPC()\n",
- " slopespc.data = slope_dict_to_arr(r)\n",
- " metadata.calibration_constant = slopespc\n",
"\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=maxcells, bias_voltage=bias_voltage,\n",
- " acquisition_rate=acq_rate, gain_setting=gain_setting)\n",
- " metadata.detector_condition = condition\n",
+ " for const in [\"SlopesPC\", \"BadPixelsPC\"]:\n",
"\n",
- " # specify the a version for this constant\n",
- " \n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=getattr(Detectors.AGIPD1M1, qm))\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=getattr(Detectors.AGIPD1M1, qm),\n",
- " start=creation_time)\n",
- " \n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface)\n",
- " print(f'Constant SlopesPC for module {qm} was injected to the calibration DB. '\n",
- " f'Begin at: {metadata.calibration_constant_version.begin_at}')\n",
- " \n",
- " # bad pixels\n",
- " \n",
- " metadata = ConstantMetaData()\n",
- " badpixpc = Constants.AGIPD.BadPixelsPC()\n",
- " badpixpc.data = bad_pixels[qm]\n",
- " metadata.calibration_constant = badpixpc\n",
- " \n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=maxcells, bias_voltage=bias_voltage,\n",
- " acquisition_rate=acq_rate, gain_setting=gain_setting)\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=getattr(Detectors.AGIPD1M1, qm))\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=getattr(Detectors.AGIPD1M1, qm),\n",
- " start=creation_time)\n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface)\n",
- " print(f'Constant BadPixelsPC for module {qm} was injected to the calibration DB. '\n",
- " f'Begin at: {metadata.calibration_constant_version.begin_at}')"
+ " dbconst = getattr(Constants.AGIPD, const)()\n",
+ "\n",
+ " # set the operating condition\n",
+ " condition = Conditions.Dark.AGIPD(memory_cells=maxcells, bias_voltage=bias_voltage,\n",
+ " acquisition_rate=acq_rate, gain_setting=gain_setting)\n",
+ "\n",
+ " if const == \"SlopesPC\":\n",
+ " dbconst.data = slope_dict_to_arr(r)\n",
+ " else:\n",
+ " dbconst.data = bad_pixels[qm]\n",
+ "\n",
+ " md = send_to_db(getattr(det, qm), dbconst, condition, file_loc, \n",
+ " cal_db_interface, creation_time=creation_time)\n",
+ "\n",
+ "print(\"Constants parameter conditions are:\\n\")\n",
+ "print(f\"• memory_cells: {maxcells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
+ " f\"• acquisition_rate: {acq_rate}\\n• gain_setting: {gain_setting}\\n\"\n",
+ " f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
},
{
diff --git a/requirements.txt b/requirements.txt
index 196c198b2..95a859441 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -34,7 +34,7 @@ pyyaml == 5.3
pyzmq == 19.0.0
scikit-learn == 0.22.2.post1
sharedmem == 0.3.7
-sphinx == 1.4.5
+sphinx == 1.8.5
tabulate == 0.8.6
unittest-xml-reporting == 3.0.2
.
diff --git a/xfel_calibrate/finalize.py b/xfel_calibrate/finalize.py
index 5c96202fb..df620c513 100644
--- a/xfel_calibrate/finalize.py
+++ b/xfel_calibrate/finalize.py
@@ -209,7 +209,7 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
out_path = report_path
try:
- check_call([sys.executable, "-m", "sphinx.quickstart",
+ check_call([sys.executable, "-m", "sphinx.cmd.quickstart",
"--quiet",
"--project='{}'".format(project),
"--author='{}'".format(author),
--
GitLab