diff --git a/notebooks/LPD/LPDChar_Darks_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
index 93a53fddc8f7499c0dc15f9e2017525e6dc1116a..8f793f13bf319d7d5ed59f78c92f1ed489b2afae 100644
--- a/notebooks/LPD/LPDChar_Darks_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
@@ -52,45 +52,42 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "in_folder = \"/gpfs/exfel/exp/FXE/201930/p900063/raw\" # path to input data, required\n",
+ "in_folder = \"/gpfs/exfel/exp/FXE/201931/p900088/raw\" # path to input data, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/karnem/LPD/\" # path to output to, required\n",
"sequences = [0] # sequence files to evaluate\n",
"modules = [-1] # list of modules to evaluate, RANGE ALLOWED\n",
"\n",
"capacitor_setting = 5 # capacitor_setting for which data was taken, required\n",
- "run_high = 358 # run number in which high gain data was recorded, required\n",
- "run_med = 359 # run number in which medium gain data was recorded, required\n",
- "run_low = 360 # run number in which low gain data was recorded, required\n",
+ "run_high = 112 # run number in which high gain data was recorded, required\n",
+ "run_med = 113 # run number in which medium gain data was recorded, required\n",
+ "run_low = 114 # run number in which low gain data was recorded, required\n",
"\n",
"mem_cells = 512 # number of memory cells used\n",
"local_output = True # output constants locally\n",
"db_output = True # output constants to database\n",
"bias_voltage = 250 # detector bias voltage\n",
- "cal_db_interface = \"tcp://max-exfl016:8017\" # the database interface to use\n",
+ "cal_db_interface = \"tcp://max-exfl016:8015#8025\" # the database interface to use\n",
+ "cal_db_timeout = 300000 # timeout on caldb requests\"\n",
"\n",
"thresholds_offset_sigma = 3. # bad pixel relative threshold in terms of n sigma offset\n",
"thresholds_offset_hard = [400, 1500] # bad pixel hard threshold\n",
"thresholds_noise_sigma = 7. # bad pixel relative threshold in terms of n sigma noise\n",
"thresholds_noise_hard = [1, 35] # bad pixel hard threshold\n",
"skip_first_ntrains = 10 # Number of first trains to skip\n",
- "use_dir_creation_date = False # use the creation date of the directory for database time derivation\n",
+ "not_use_dir_creation_date = False # do not use the creation date of the directory for database time derivation\n",
"instrument = \"FXE\" # instrument name\n",
- "ntrains = 10 # number of trains to use\n",
+ "ntrains = 300 # number of trains to use\n",
"high_res_badpix_3d = False # plot bad-pixel summary in high resolution"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"from collections import OrderedDict\n",
@@ -121,7 +118,8 @@
"from cal_tools.tools import (gain_map_files, parse_runs, \n",
" run_prop_seq_from_path, \n",
" get_notebook_name, \n",
- " get_dir_creation_date, get_from_db)\n",
+ " get_dir_creation_date, get_from_db,\n",
+ " get_random_db_interface)\n",
"from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import (show_overview, plot_badpix_3d, \n",
@@ -131,9 +129,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"client = Client(profile=cluster_profile)\n",
@@ -165,15 +161,18 @@
"DET_FILE_INSET = \"LPD\"\n",
"\n",
"creation_time = None\n",
- "if use_dir_creation_date:\n",
+ "if not not_use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run_high)\n",
" print(\"Using {} as creation time\".format(creation_time))\n",
"\n",
"run, prop, seq = run_prop_seq_from_path(in_folder)\n",
"logger = InfluxLogger(detector=\"LPD\", instrument=instrument, mem_cells=mem_cells,\n",
" notebook=get_notebook_name(), proposal=prop)\n",
- " \n",
+ "\n",
+ "cal_db_interface = get_random_db_interface(cal_db_interface)\n",
+ "\n",
"display(Markdown('## Evaluated parameters'))\n",
+ "print('CalDB Interface {}'.format(cal_db_interface))\n",
"print(\"Proposal: {}\".format(prop))\n",
"print(\"Memory cells: {}/{}\".format(mem_cells, max_cells))\n",
"print(\"Runs: {}, {}, {}\".format(run_high, run_med, run_low))\n",
@@ -187,9 +186,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# set everything up filewise\n",
@@ -210,9 +207,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# the actual characterization - to not eded this without consultation\n",
@@ -335,13 +330,13 @@
" (offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
" noise_g[cap][qm] = np.zeros_like(offset_g[cap][qm])\n",
" badpix_g[cap][qm] = np.zeros_like(offset_g[cap][qm])\n",
- " data_g[cap][qm] = np.zeros((data.shape[0], 3))\n",
+ " data_g[cap][qm] = np.zeros((ntrains, 3))\n",
" ntest_g[cap][qm] = np.zeros_like(offset_g[cap][qm])\n",
"\n",
" offset_g[cap][qm][..., gg] = offset\n",
" noise_g[cap][qm][..., gg] = noise\n",
" badpix_g[cap][qm][..., gg] = bp\n",
- " data_g[cap][qm][..., gg] = data\n",
+ " data_g[cap][qm][:data.shape[0], gg] = data\n",
" ntest_g[cap][qm][..., gg] = normal\n",
"\n",
" hn, cn = np.histogram(data, bins=20)\n",
@@ -360,9 +355,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# save everything to file.\n",
@@ -386,16 +379,14 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Retrieve existing constants for comparison\n",
"clist = [\"Offset\", \"Noise\", \"BadPixelsDark\"]\n",
"old_const = {}\n",
"old_mdata = {}\n",
- "creation_time = dateutil.parser.parse(\"2019-02-14\")\n",
+ "\n",
"print('Retrieve pre-existing constants for comparison.')\n",
"for cap in capacitor_settings:\n",
" for qm in offset_g[cap].keys():\n",
@@ -409,7 +400,7 @@
" condition,\n",
" None,\n",
" cal_db_interface, creation_time=creation_time,\n",
- " verbosity=2, timeout=30000)\n",
+ " verbosity=2, timeout=cal_db_timeout)\n",
"\n",
" old_const[const] = data\n",
"\n",
@@ -423,9 +414,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"res = OrderedDict()\n",
@@ -443,9 +432,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Save constants in the calibration DB\n",
@@ -474,7 +461,7 @@
" else:\n",
" metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
" start=creation_time)\n",
- " metadata.send(cal_db_interface)\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
" msg = 'Const {} for module {} was injected to the calibration DB. Begin at: {}'\n",
" print(msg.format(const, qm,\n",
" metadata.calibration_constant_version.begin_at))"
@@ -483,9 +470,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"qm = \"Q{}M{}\".format(modules[0]//4+1, modules[0] % 4+1)\n",
@@ -557,9 +542,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig, grid = plt.subplots(3, 1, sharex=\"col\", sharey=\"row\", figsize=(10, 7))\n",
@@ -648,9 +631,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Loop over capacitor settings, modules, constants\n",
@@ -723,7 +704,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
"scrolled": true
},
"outputs": [],
@@ -828,9 +808,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"cols = {BadPixels.NOISE_OUT_OF_THRESHOLD.value: (BadPixels.NOISE_OUT_OF_THRESHOLD.name, '#FF000080'),\n",
@@ -872,9 +850,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"display(Markdown('The following pre-existing constants are used for comparison: \\n'))\n",
@@ -885,9 +861,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Loop over capacitor settings, modules, constants\n",
@@ -983,9 +957,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Loop over capacitor settings, modules, constants\n",
@@ -1018,9 +990,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Loop over capacitor settings, modules, constants\n",
@@ -1074,9 +1044,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Loop over capacitor settings, modules, constants\n",
@@ -1166,9 +1134,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"table = []\n",
@@ -1222,9 +1188,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"\n",
@@ -1280,7 +1244,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.4.3"
+ "version": "3.6.7"
}
},
"nbformat": 4,
diff --git a/notebooks/LPD/LPDChar_Darks_Summary_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_Summary_NBC.ipynb
index 136a8e3241cf4774994205867e593c8ff90104a6..cb30454e34b6b65d67573cbebb728192e8a72fdc 100644
--- a/notebooks/LPD/LPDChar_Darks_Summary_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_Summary_NBC.ipynb
@@ -10,9 +10,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
@@ -22,9 +20,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"from collections import OrderedDict\n",
@@ -34,6 +30,7 @@
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
+ "import glob\n",
"import h5py\n",
"from IPython.display import display, Markdown, Latex\n",
"import numpy as np\n",
@@ -52,9 +49,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Load constants from local files\n",
@@ -81,9 +76,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"mod_idx = np.argsort(mod_names)\n",
@@ -102,9 +95,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"display(Markdown('## Processed modules ##'))\n",
@@ -157,9 +148,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"q_pad = 15\n",
@@ -214,7 +203,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
"scrolled": true
},
"outputs": [],
@@ -296,9 +284,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"if u'$' in tabulate.LATEX_ESCAPE_RULES:\n",
@@ -311,9 +297,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"header = ['Module', 'High gain', 'Medium gain', 'Low gain']\n",
@@ -373,7 +357,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.4.3"
+ "version": "3.6.7"
}
},
"nbformat": 4,