diff --git a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
index f08ef2fad6e21961ce4cfbd1a44e61e05a63ff6b..eb7b7721c105dc12548ac3a1fb181961ecde740c 100644
--- a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
+++ b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
@@ -20,17 +20,16 @@
"ExecuteTime": {
"end_time": "2019-02-20T12:42:51.255184Z",
"start_time": "2019-02-20T12:42:51.225500Z"
- },
- "collapsed": true
+ }
},
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "in_folder = \"/gpfs/exfel/exp/SCS/201930/p900079/raw\" # path to input data, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/haufs/test/\" # path to output to, required\n",
+ "in_folder = \"/gpfs/exfel/exp/SCS/201931/p900095/raw\" # path to input data, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/\" # path to output to, required\n",
"sequences = [0] # sequence files to evaluate.\n",
"\n",
- "run = 20 # run number in which data was recorded, required\n",
+ "run = 1417 # run number in which data was recorded, required\n",
"\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
"local_output = False # output constants locally\n",
@@ -48,7 +47,7 @@
"\n",
"instrument = \"SCS\" # the instrument\n",
"high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. Runtime: ~ 1h\n",
- "modules = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] # module to run for"
+ "modules = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] # modules to run for"
]
},
{
@@ -197,6 +196,10 @@
" import h5py\n",
" from cal_tools.enums import BadPixels\n",
" \n",
+ " from hashlib import blake2b\n",
+ " import struct\n",
+ " import binascii\n",
+ " \n",
" def get_num_cells(fname, loc, module):\n",
" with h5py.File(fname, \"r\") as f:\n",
"\n",
@@ -212,7 +215,9 @@
"\n",
" if cells == 0:\n",
" cells = get_num_cells(filename, loc, channel)\n",
- " \n",
+ "\n",
+ " pulseid_checksum = None\n",
+ "\n",
" thresholds_offset_hard, thresholds_offset_sigma, thresholds_noise_hard, thresholds_noise_sigma = bp_thresh \n",
"\n",
" infile = h5py.File(filename, \"r\", driver=\"core\")\n",
@@ -221,6 +226,9 @@
" first = np.squeeze(infile[\"/INDEX/{}/DET/{}CH0:xtdf/image/first\".format(loc, channel)])\n",
" last_index = int(first[count != 0][-1]+count[count != 0][-1])\n",
" first_index = int(first[count != 0][0])\n",
+ " pulseids = infile[\"INSTRUMENT/{}/DET/{}CH0:xtdf/image/pulseId\".format(loc, channel)][first_index:int(first[count != 0][1])]\n",
+ " bveto = blake2b(pulseids.data, digest_size=8)\n",
+ " pulseid_checksum = struct.unpack('d', binascii.unhexlify(bveto.hexdigest()))[0]\n",
" else:\n",
" status = np.squeeze(infile[\"/INDEX/{}/DET/{}CH0:xtdf/image/status\".format(loc, channel)])\n",
" if np.count_nonzero(status != 0) == 0:\n",
@@ -271,7 +279,7 @@
" bp[~np.isfinite(noise)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
"\n",
"\n",
- " return offset, noise, bp, cells\n",
+ " return offset, noise, bp, cells, pulseid_checksum\n",
" \n",
" \n",
"offset_g = OrderedDict()\n",
@@ -282,6 +290,7 @@
"\n",
"start = datetime.now()\n",
"all_cells = []\n",
+ "checksums = {}\n",
"\n",
"for gain, mapped_files in gain_mapped_files.items():\n",
" \n",
@@ -304,7 +313,7 @@
" \n",
" for ii, r in enumerate(results):\n",
" i = modules[ii]\n",
- " offset, noise, bp, thiscell = r\n",
+ " offset, noise, bp, thiscell, pulseid_checksum = r\n",
" all_cells.append(thiscell)\n",
" qm = \"Q{}M{}\".format(i//4 +1, i % 4 + 1)\n",
" if qm not in offset_g:\n",
@@ -312,7 +321,8 @@
" noise_g[qm] = np.zeros_like(offset_g[qm])\n",
" \n",
" badpix_g[qm] = np.zeros_like(offset_g[qm], np.uint32)\n",
- " \n",
+ " checksums[qm] = pulseid_checksum\n",
+ " \n",
" offset_g[qm][...] = offset\n",
" noise_g[qm][...] = noise\n",
" badpix_g[qm][...] = bp\n",
@@ -334,8 +344,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T09:38:18.234582Z",
"start_time": "2018-12-06T09:38:18.222838Z"
- },
- "collapsed": true
+ }
},
"outputs": [],
"source": [
@@ -364,66 +373,72 @@
"ExecuteTime": {
"end_time": "2018-12-06T09:49:32.449330Z",
"start_time": "2018-12-06T09:49:20.231607Z"
- },
- "collapsed": true
+ }
},
"outputs": [],
"source": [
"if db_output:\n",
- " for qm in offset_g.keys():\n",
- " metadata = ConstantMetaData()\n",
- " offset = Constants.DSSC.Offset()\n",
- " offset.data = offset_g[qm]\n",
- " metadata.calibration_constant = offset\n",
- "\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.DSSC(memory_cells=max_cells, bias_voltage=bias_voltage)\n",
- " detinst = getattr(Detectors, dinstance)\n",
- "\n",
- " device = getattr(detinst, qm)\n",
- " \n",
- " metadata.detector_condition = condition\n",
- " \n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " metadata.send(cal_db_interface, timeout=3000000)\n",
- " \n",
- " \n",
- " metadata = ConstantMetaData()\n",
- " noise = Constants.DSSC.Noise()\n",
- " noise.data = noise_g[qm]\n",
- " metadata.calibration_constant = noise\n",
- "\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.DSSC(memory_cells=max_cells, bias_voltage=bias_voltage)\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " metadata.send(cal_db_interface, timeout=3000000)\n",
- " \n",
- " continue # no bad pixels yet\n",
- " metadata = ConstantMetaData()\n",
- " badpixels = Constants.DSSC.BadPixelsDark()\n",
- " badpixels.data = badpix_g[qm]\n",
- " metadata.calibration_constant = badpixels\n",
- "\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.DSSC(memory_cells=max_cells, bias_voltage=bias_voltage)\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " metadata.send(cal_db_interface, timeout=3000000)"
+ " for dont_use_pulseIds in [True, False]:\n",
+ " for qm in offset_g.keys():\n",
+ " try:\n",
+ " metadata = ConstantMetaData()\n",
+ " offset = Constants.DSSC.Offset()\n",
+ " offset.data = offset_g[qm]\n",
+ " metadata.calibration_constant = offset\n",
+ " pidsum = None if dont_use_pulseIds else checksums[qm]\n",
+ " # set the operating condition\n",
+ " condition = Conditions.Dark.DSSC(memory_cells=max_cells, bias_voltage=bias_voltage,\n",
+ " pulseid_checksum=pidsum)\n",
+ " detinst = getattr(Detectors, dinstance)\n",
+ "\n",
+ " device = getattr(detinst, qm)\n",
+ "\n",
+ " metadata.detector_condition = condition\n",
+ "\n",
+ " # specify the a version for this constant\n",
+ " if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
+ " metadata.send(cal_db_interface, timeout=3000000)\n",
+ "\n",
+ "\n",
+ " metadata = ConstantMetaData()\n",
+ " noise = Constants.DSSC.Noise()\n",
+ " noise.data = noise_g[qm]\n",
+ " metadata.calibration_constant = noise\n",
+ "\n",
+ " # set the operating condition\n",
+ " condition = Conditions.Dark.DSSC(memory_cells=max_cells, bias_voltage=bias_voltage,\n",
+ " pulseid_checksum=pidsum)\n",
+ " metadata.detector_condition = condition\n",
+ "\n",
+ " # specify the a version for this constant\n",
+ " if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
+ " metadata.send(cal_db_interface, timeout=3000000)\n",
+ "\n",
+ " continue # no bad pixels yet\n",
+ " metadata = ConstantMetaData()\n",
+ " badpixels = Constants.DSSC.BadPixelsDark()\n",
+ " badpixels.data = badpix_g[qm]\n",
+ " metadata.calibration_constant = badpixels\n",
+ "\n",
+ " # set the operating condition\n",
+ " condition = Conditions.Dark.DSSC(memory_cells=max_cells, bias_voltage=bias_voltage,\n",
+ " pulseid_checksum=pidsum)\n",
+ " metadata.detector_condition = condition\n",
+ "\n",
+ " # specify the a version for this constant\n",
+ " if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
+ " metadata.send(cal_db_interface, timeout=3000000)\n",
+ " except Exception as e:\n",
+ " print(e)"
]
},
{
@@ -435,6 +450,16 @@
"Single cell overviews allow to identify potential effects on all memory cells, e.g. on sensor level. Additionally, they should serve as a first sanity check on expected behaviour, e.g. if structuring on the ASIC level is visible in the offsets, but otherwise no immediate artifacts are visible."
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for r in res.values():\n",
+ " del r[\"BadPixels\"]"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -447,7 +472,7 @@
},
"outputs": [],
"source": [
- "cell = 3\n",
+ "cell = 9\n",
"gain = 0\n",
"out_folder = None\n",
"show_overview(res, cell, gain, out_folder=out_folder, infix=\"_\".join(offset_runs.values()))"
@@ -466,7 +491,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true,
"scrolled": false
},
"outputs": [],
@@ -496,7 +520,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true,
"scrolled": false
},
"outputs": [],
@@ -509,7 +532,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true,
"scrolled": false
},
"outputs": [],
@@ -521,9 +543,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"bad_pixel_aggregate_g = OrderedDict()\n",
@@ -536,9 +556,14 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
"outputs": [],
"source": []
}