diff --git a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
index 6e6da8c8c21c74591d6003c9a3931e4122ccaa80..f08ef2fad6e21961ce4cfbd1a44e61e05a63ff6b 100644
--- a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
+++ b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
@@ -58,8 +58,7 @@
"ExecuteTime": {
"end_time": "2019-02-20T12:42:52.599660Z",
"start_time": "2019-02-20T12:42:51.472138Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -127,8 +126,7 @@
"ExecuteTime": {
"end_time": "2019-02-20T12:42:52.608214Z",
"start_time": "2019-02-20T12:42:52.601257Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -157,8 +155,7 @@
"ExecuteTime": {
"end_time": "2019-02-20T12:42:54.024731Z",
"start_time": "2019-02-20T12:42:53.901555Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -188,8 +185,7 @@
"ExecuteTime": {
"end_time": "2019-02-20T10:50:55.839958Z",
"start_time": "2019-02-20T10:50:55.468134Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -207,7 +203,8 @@
" cells = f[\"INSTRUMENT/{}/DET/{}CH0:xtdf/image/cellId\".format(loc, module)][()]\n",
" maxcell = np.max(cells)\n",
" options = [100, 200, 400, 500, 600, 700, 800]\n",
- " dists = [abs(o-maxcell) for o in options]\n",
+ " dists = np.array([(o-maxcell) for o in options])\n",
+ " dists[dists<0] = 10000 # assure to always go higher\n",
" return options[np.argmin(dists)]\n",
" \n",
" filename, filename_out, channel = inp\n",
@@ -446,7 +443,6 @@
"end_time": "2018-12-06T09:49:14.540552Z",
"start_time": "2018-12-06T09:49:13.009674Z"
},
- "collapsed": false,
"scrolled": false
},
"outputs": [],
@@ -563,7 +559,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.6"
+ "version": "3.6.7"
}
},
"nbformat": 4,
diff --git a/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb b/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb
index d3782ce5563b67a37bfc6bd3f941efac3f0148bf..e9d65ef342941b2fe2c7617189e4c07c3fb28012 100644
--- a/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb
+++ b/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb
@@ -18,32 +18,31 @@
"ExecuteTime": {
"end_time": "2019-02-21T11:30:06.730220Z",
"start_time": "2019-02-21T11:30:06.658286Z"
- },
- "collapsed": true
+ }
},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/SCS/201802/p002252/raw/\" # the folder to read data from, required\n",
- "run = 20 # runs to process, required\n",
+ "in_folder = \"/gpfs/exfel/exp/SCS/201901/p002212/raw/\" # the folder to read data from, required\n",
+ "run = 29 # runs to process, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/xcal/test/\" # the folder to output to, required\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
"overwrite = True # set to True if existing data should be overwritten\n",
"cluster_profile = \"noDB\" # cluster profile to use\n",
"max_pulses = 500 # maximum number of pulses per train\n",
- "bias_voltage = 300 # detector bias voltage\n",
+ "bias_voltage = 100 # detector bias voltage\n",
"cal_db_interface = \"tcp://max-exfl016:8020#8025\" # the database interface to use\n",
"use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
"sequences_per_node = 2 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel\n",
- "cal_db_timeout = 30000 # in milli seconds\n",
+ "cal_db_timeout = 300000 # in milli seconds\n",
"chunk_size_idim = 1 # chunking size of imaging dimension, adjust if user software is sensitive to this.\n",
"instrument = \"SCS\" # the instrument the detector is installed at, required\n",
"mask_noisy_asic = 0.25 # set to a value other than 0 and below 1 to mask entire ADC if fraction of noisy pixels is above\n",
- "offset_image = \"-1\" # last one\n",
+ "offset_image = \"PP\" # last one\n",
"mask_cold_asic = 0.25 # mask cold ASICS if number of pixels with negligable standard deviation is larger than this fraction\n",
"noisy_pix_threshold = 1. # threshold above which ap pixel is considered noisy.\n",
"geo_file = \"/gpfs/exfel/data/scratch/xcal/dssc_geo_june19.h5\" # detector geometry file\n",
- "\n",
+ "dinstance = \"DSSC1M1\"\n",
"\n",
"def balance_sequences(in_folder, run, sequences, sequences_per_node):\n",
" import glob\n",
@@ -74,8 +73,7 @@
"ExecuteTime": {
"end_time": "2019-02-21T11:30:07.086286Z",
"start_time": "2019-02-21T11:30:06.929722Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -140,7 +138,8 @@
" dinstance = \"DSSC1M1\"\n",
"print(\"Detector in use is {}\".format(loc)) \n",
"\n",
- "offset_image = int(offset_image)"
+ "if offset_image.upper() != \"PP\":\n",
+ " offset_image = int(offset_image)"
]
},
{
@@ -150,8 +149,7 @@
"ExecuteTime": {
"end_time": "2019-02-21T11:30:07.263445Z",
"start_time": "2019-02-21T11:30:07.217070Z"
- },
- "collapsed": true
+ }
},
"outputs": [],
"source": [
@@ -222,8 +220,7 @@
"ExecuteTime": {
"end_time": "2019-02-21T11:30:07.974174Z",
"start_time": "2019-02-21T11:30:07.914832Z"
- },
- "collapsed": true
+ }
},
"outputs": [],
"source": [
@@ -289,8 +286,7 @@
"ExecuteTime": {
"end_time": "2019-02-21T11:30:08.870802Z",
"start_time": "2019-02-21T11:30:08.826285Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -311,7 +307,8 @@
" table.append((ti, \"\", i, f))\n",
" i += 1\n",
" ti += 1\n",
- "md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=[\"#\", \"module\", \"# module\", \"file\"]))) \n",
+ "if len(table):\n",
+ " md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=[\"#\", \"module\", \"# module\", \"file\"]))) \n",
"# restore the queue\n",
"mapped_files, mod_ids, total_sequences, sequences_qm, one_module = map_modules_from_files(file_list)"
]
@@ -323,19 +320,21 @@
"ExecuteTime": {
"end_time": "2019-02-21T11:30:16.057429Z",
"start_time": "2019-02-21T11:30:10.082114Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
"import copy\n",
"from functools import partial\n",
"def correct_module(total_sequences, sequences_qm, loc, dinstance, offset_image,\n",
- " mask_noisy_asic, mask_cold_asic, noisy_pix_threshold, chunksize, inp):\n",
+ " mask_noisy_asic, mask_cold_asic, noisy_pix_threshold, chunksize,\n",
+ " mem_cells, bias_voltage, cal_db_timeout, creation_time, cal_db_interface, inp):\n",
" import numpy as np\n",
" import copy\n",
" import h5py\n",
" from cal_tools.enums import BadPixels\n",
+ " from cal_tools.tools import get_dir_creation_date, get_constant_from_db_and_time, get_random_db_interface\n",
+ " from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
" \n",
" filename, filename_out, channel, qm = inp\n",
" h5path = \"INSTRUMENT/{}/DET/{}CH0:xtdf/\".format(loc, channel)\n",
@@ -347,6 +346,40 @@
" hists_signal_high = None\n",
" pulse_edges = None\n",
" \n",
+ " def get_num_cells(fname, loc, module):\n",
+ " with h5py.File(fname, \"r\") as f:\n",
+ "\n",
+ " cells = f[\"INSTRUMENT/{}/DET/{}CH0:xtdf/image/cellId\".format(loc, module)][()]\n",
+ " maxcell = np.max(cells)\n",
+ " options = [100, 200, 400, 500, 600, 700, 800]\n",
+ " dists = np.array([(o-maxcell) for o in options])\n",
+ " dists[dists<0] = 10000 # assure to always go higher\n",
+ " return options[np.argmin(dists)]\n",
+ " \n",
+ "\n",
+ " if mem_cells == 0:\n",
+ " mem_cells = get_num_cells(filename, loc, channel)\n",
+ " \n",
+ " print(\"Memcells: {}\".format(mem_cells))\n",
+ " \n",
+ " condition = Conditions.Dark.DSSC(bias_voltage=bias_voltage, memory_cells=mem_cells)\n",
+ " \n",
+ " \n",
+ " detinst = getattr(Detectors, dinstance)\n",
+ " device = getattr(detinst, qm)\n",
+ " offset, when = get_constant_from_db_and_time(device,\n",
+ " Constants.DSSC.Offset(),\n",
+ " condition,\n",
+ " None,\n",
+ " get_random_db_interface(cal_db_interface),\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout)\n",
+ " if offset is not None:\n",
+ " \n",
+ " offset = np.moveaxis(np.moveaxis(offset[...], 2, 0), 2, 1)\n",
+ " print(offset.shape)\n",
+ " \n",
+ " \n",
" def copy_and_sanitize_non_cal_data(infile, outfile):\n",
" # these are touched in the correct function, do not copy them here\n",
" dont_copy = [\"data\"]\n",
@@ -380,7 +413,7 @@
" chunks = (chunksize, oshape[1], oshape[2])\n",
" ddset = outfile.create_dataset(h5path + \"image/data\",\n",
" oshape, chunks=chunks,\n",
- " dtype=np.uint16,\n",
+ " dtype=np.float32,\n",
" fletcher32=True)\n",
" \n",
" mdset = outfile.create_dataset(h5path + \"image/mask\",\n",
@@ -395,8 +428,12 @@
" first = first_arr[train]\n",
" last = last_arr[train]\n",
" data = np.squeeze(infile[h5path+\"image/data\"][first:last, ...].astype(np.float32))\n",
+ " cellId = np.squeeze(infile[h5path+\"image/cellId\"][first:last, ...])\n",
" pulseId = np.squeeze(infile[h5path+\"image/pulseId\"][first:last, ...])\n",
- " data -= data[offset_image, ...]\n",
+ " if offset_image != \"PP\" or offset is None:\n",
+ " data -= data[offset_image, ...]\n",
+ " else:\n",
+ " data -= offset[cellId,...]\n",
" \n",
" if train == 0:\n",
" pulseId = np.repeat(pulseId[:, None], data.shape[1], axis=1)\n",
@@ -413,8 +450,8 @@
" pulseId.flatten(),\n",
" bins=bins,\n",
" range=rnge) \n",
- " data[data < 0] = 0\n",
- " ddset[first:last, ...] = data.astype(np.uint16)\n",
+ " # data[data < 0] = 0\n",
+ " ddset[first:last, ...] = data\n",
" \n",
" # find static and noisy values in dark images\n",
" data = infile[h5path+\"image/data\"][last, ...].astype(np.float32)\n",
@@ -444,7 +481,7 @@
" \n",
" \n",
" \n",
- " return (hists_signal_low, hists_signal_high, low_edges, high_edges, pulse_edges)\n",
+ " return (hists_signal_low, hists_signal_high, low_edges, high_edges, pulse_edges, when, qm)\n",
" \n",
"done = False\n",
"first_files = []\n",
@@ -456,7 +493,8 @@
"\n",
"low_edges, high_edges, pulse_edges = None, None, None\n",
"\n",
- "\n",
+ "whens = []\n",
+ "qms = []\n",
"while not done:\n",
" \n",
" dones = []\n",
@@ -479,7 +517,8 @@
" print(\"Running {} tasks parallel\".format(len(inp)))\n",
" p = partial(correct_module, total_sequences, sequences_qm,\n",
" loc, dinstance, offset_image, mask_noisy_asic,\n",
- " mask_cold_asic, noisy_pix_threshold, chunk_size_idim)\n",
+ " mask_cold_asic, noisy_pix_threshold, chunk_size_idim,\n",
+ " mem_cells, bias_voltage, cal_db_timeout, creation_time, cal_db_interface)\n",
" \n",
" r = view.map_sync(p, inp)\n",
" #r = list(map(p, inp))\n",
@@ -488,13 +527,19 @@
" \n",
" for rr in r:\n",
" if rr is not None:\n",
- " hl, hh, low_edges, high_edges, pulse_edges = rr \n",
+ " hl, hh, low_edges, high_edges, pulse_edges, when, qm = rr \n",
+ " whens.append(when)\n",
+ " qms.append(qm)\n",
" if hl is not None: # any one being None will also make the others None\n",
" hists_signal_low += hl.astype(np.float64)\n",
" hists_signal_high += hh.astype(np.float64) \n",
" \n",
" done = all(dones)\n",
- "\n"
+ "\n",
+ "whens = [x for _,x in sorted(zip(qms,whens))]\n",
+ "qms = sorted(qms)\n",
+ "for i, qm in enumerate(qms):\n",
+ " print(\"Offset for {} was injected on {}\".format(qm, whens[i].isoformat()))"
]
},
{
@@ -504,8 +549,7 @@
"ExecuteTime": {
"end_time": "2019-02-18T17:28:51.765030Z",
"start_time": "2019-02-18T17:28:51.714783Z"
- },
- "collapsed": true
+ }
},
"outputs": [],
"source": [
@@ -541,8 +585,7 @@
"ExecuteTime": {
"end_time": "2019-02-18T17:28:53.690522Z",
"start_time": "2019-02-18T17:28:52.860143Z"
- },
- "collapsed": true
+ }
},
"outputs": [],
"source": [
@@ -577,7 +620,6 @@
"end_time": "2019-02-18T17:28:57.327702Z",
"start_time": "2019-02-18T17:28:54.377061Z"
},
- "collapsed": false,
"scrolled": false
},
"outputs": [],
@@ -595,8 +637,7 @@
"ExecuteTime": {
"end_time": "2019-02-18T17:29:20.634480Z",
"start_time": "2019-02-18T17:28:57.329231Z"
- },
- "collapsed": true
+ }
},
"outputs": [],
"source": [
@@ -638,9 +679,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"def combine_stack(d, sdim):\n",
@@ -702,8 +741,7 @@
"ExecuteTime": {
"end_time": "2019-02-18T17:29:27.025667Z",
"start_time": "2019-02-18T17:29:20.642029Z"
- },
- "collapsed": true
+ }
},
"outputs": [],
"source": [
@@ -729,8 +767,7 @@
"ExecuteTime": {
"end_time": "2019-02-18T17:29:33.226396Z",
"start_time": "2019-02-18T17:29:27.027758Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -760,8 +797,7 @@
"ExecuteTime": {
"end_time": "2019-02-18T17:29:33.761015Z",
"start_time": "2019-02-18T17:29:33.227922Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -781,8 +817,7 @@
"ExecuteTime": {
"end_time": "2019-02-18T17:29:35.903487Z",
"start_time": "2019-02-18T17:29:33.762568Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -807,8 +842,7 @@
"ExecuteTime": {
"end_time": "2019-02-18T17:29:39.369686Z",
"start_time": "2019-02-18T17:29:35.905152Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -833,18 +867,14 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"\n",
@@ -863,8 +893,7 @@
"ExecuteTime": {
"end_time": "2019-02-18T17:29:49.217848Z",
"start_time": "2019-02-18T17:29:39.371232Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -891,8 +920,7 @@
"ExecuteTime": {
"end_time": "2019-02-18T17:29:49.651913Z",
"start_time": "2019-02-18T17:29:49.643556Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -921,8 +949,7 @@
"ExecuteTime": {
"end_time": "2019-02-18T17:29:51.686562Z",
"start_time": "2019-02-18T17:29:50.088883Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -947,8 +974,7 @@
"ExecuteTime": {
"end_time": "2019-02-18T17:29:53.662423Z",
"start_time": "2019-02-18T17:29:51.688376Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -962,18 +988,14 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"outputs": [],
"source": []
}
@@ -994,7 +1016,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.6"
+ "version": "3.6.7"
}
},
"nbformat": 4,